Cost Monitoring LLM Golang CI: Budget Enforcement, Alert, Per-Feature Tracking

 1Perbedaan fundamental: infrastructure cost vs LLM cost
 2
 3Infrastructure (Kubernetes, RDS):
 4  Cost: relatif predictable, naik linear dengan traffic
 5  Variability: rendah — harga per instance stabil
 6  Anomaly: mudah dideteksi (CPU spike, disk usage)
 7
 8LLM API cost:
 9  Cost: dapat naik 100× dari normal dalam hitungan menit
10  Variability: sangat tinggi — bergantung token count
11  Anomaly: sulit dideteksi tanpa monitoring khusus
12
13Contoh nyata:
14  Normal: 1000 requests/hari × $0.001 = $1/hari
15
16  Bug skenario A (prompt loop):
17  1000 requests/menit × 60 × 24 = 1,440,000 requests/hari
18  Cost: $1,440/hari → $43,200/bulan (unexpected!)
19
20  Bug skenario B (prompt terlalu besar):
21  1000 req/hari × $0.10/req (large prompt) = $100/hari
22  Normal expected: $1/hari

 1// internal/ai/cost/calculator.go
 2
 3package cost
 4
 5// Harga Anthropic Claude models (per million tokens, USD)
 6var modelPricing = map[string]Pricing{
 7    "claude-haiku-4-5-20251001": {
 8        InputPerMillion:  0.80,
 9        OutputPerMillion: 4.00,
10    },
11    "claude-sonnet-4-6": {
12        InputPerMillion:  3.00,
13        OutputPerMillion: 15.00,
14    },
15    "claude-opus-4-6": {
16        InputPerMillion:  15.00,
17        OutputPerMillion: 75.00,
18    },
19}
20
21type Pricing struct {
22    InputPerMillion  float64
23    OutputPerMillion float64
24}
25
26type Usage struct {
27    Model        string
28    InputTokens  int64
29    OutputTokens int64
30    Feature      string // e.g., "pr-review", "pr-description", "spec-audit"
31}
32
33type CostCalculator struct{}
34
35func (c *CostCalculator) Calculate(usage Usage) (float64, error) {
36    pricing, ok := modelPricing[usage.Model]
37    if !ok {
38        return 0, fmt.Errorf("calculate: unknown model %q", usage.Model)
39    }
40
41    inputCost  := float64(usage.InputTokens) / 1_000_000 * pricing.InputPerMillion
42    outputCost := float64(usage.OutputTokens) / 1_000_000 * pricing.OutputPerMillion
43
44    return inputCost + outputCost, nil
45}
46
47// Example:
48// haiku: 1500 input + 400 output = (1500×0.80 + 400×4.00) / 1,000,000
49//      = (1200 + 1600) / 1,000,000 = $0.0028 per request

 1// internal/ai/cost/tracker.go
 2
 3package cost
 4
 5import (
 6    "context"
 7    "sync"
 8    "time"
 9
10    "github.com/prometheus/client_golang/prometheus"
11)
12
13type Tracker struct {
14    mu         sync.RWMutex
15    calc       *CostCalculator
16    daily      map[string]float64  // feature → daily cost USD
17    monthly    map[string]float64  // feature → monthly cost USD
18    budgets    map[string]float64  // feature → monthly budget USD
19    lastReset  time.Time
20}
21
22func NewTracker(budgets map[string]float64) *Tracker {
23    return &Tracker{
24        calc:      &CostCalculator{},
25        daily:     make(map[string]float64),
26        monthly:   make(map[string]float64),
27        budgets:   budgets,
28        lastReset: time.Now(),
29    }
30}
31
32func (t *Tracker) Record(ctx context.Context, usage Usage) (float64, error) {
33    cost, err := t.calc.Calculate(usage)
34    if err != nil {
35        return 0, fmt.Errorf("tracker.Record: %w", err)
36    }
37
38    t.mu.Lock()
39    defer t.mu.Unlock()
40
41    t.daily[usage.Feature]   += cost
42    t.monthly[usage.Feature] += cost
43
44    // Prometheus metrics
45    aiCostTotal.WithLabelValues(usage.Feature, usage.Model).Add(cost)
46    aiTokensTotal.WithLabelValues(usage.Feature, "input").Add(float64(usage.InputTokens))
47    aiTokensTotal.WithLabelValues(usage.Feature, "output").Add(float64(usage.OutputTokens))
48
49    return cost, nil
50}
51
52func (t *Tracker) CheckBudget(feature string) (used, budget float64, exceeded bool) {
53    t.mu.RLock()
54    defer t.mu.RUnlock()
55
56    used   = t.monthly[feature]
57    budget = t.budgets[feature]
58    exceeded = budget > 0 && used >= budget
59    return
60}
61
62// Prometheus metrics
63var (
64    aiCostTotal = prometheus.NewCounterVec(
65        prometheus.CounterOpts{
66            Name: "ai_cost_usd_total",
67            Help: "Total LLM API cost in USD",
68        },
69        []string{"feature", "model"},
70    )
71    aiTokensTotal = prometheus.NewCounterVec(
72        prometheus.CounterOpts{
73            Name: "ai_tokens_total",
74            Help: "Total LLM tokens consumed",
75        },
76        []string{"feature", "token_type"},
77    )
78)

 1// internal/ai/cost/budget.go
 2
 3package cost
 4
 5import (
 6    "context"
 7    "fmt"
 8)
 9
10type BudgetError struct {
11    Feature  string
12    Used     float64
13    Budget   float64
14}
15
16func (e *BudgetError) Error() string {
17    return fmt.Sprintf("budget exceeded for %s: $%.4f used of $%.4f budget",
18        e.Feature, e.Used, e.Budget)
19}
20
21// BudgetGuard wraps any AI call with budget check
22func (t *Tracker) WithBudget(feature string, fn func() (Usage, error)) (Usage, error) {
23    // Pre-check: estimate cost sebelum call
24    used, budget, exceeded := t.CheckBudget(feature)
25    if exceeded {
26        return Usage{}, &BudgetError{
27            Feature: feature,
28            Used:    used,
29            Budget:  budget,
30        }
31    }
32
33    // Execute AI call
34    usage, err := fn()
35    if err != nil {
36        return usage, err
37    }
38
39    // Record actual cost
40    if _, err := t.Record(context.Background(), usage); err != nil {
41        // Log but don't fail — cost tracking failure shouldn't fail the request
42        fmt.Printf("cost tracking failed: %v\n", err)
43    }
44
45    return usage, nil
46}

 1# config/ai-budgets.yaml
 2
 3budgets:
 4  # CI/CD automation (per month, USD)
 5  pr-review:
 6    monthly: 5.00
 7    daily: 0.50
 8    alert_at: 0.80  # alert ketika 80% budget terpakai
 9
10  pr-description:
11    monthly: 2.00
12    daily: 0.20
13    alert_at: 0.80
14
15  spec-audit:
16    monthly: 3.00
17    daily: 0.30
18    alert_at: 0.80
19
20  release-notes:
21    monthly: 1.00
22    daily: 0.20
23    alert_at: 0.90
24
25  # Production AI features
26  ai-order-review:
27    monthly: 10.00
28    daily: 1.00
29    alert_at: 0.75
30
31  ai-product-description:
32    monthly: 15.00
33    daily: 2.00
34    alert_at: 0.75
35
36  # Total budget (semua features)
37  total:
38    monthly: 50.00
39    alert_at: 0.80

 1# .github/workflows/cost-report.yml
 2name: Daily AI Cost Report
 3
 4on:
 5  schedule:
 6    - cron: '0 1 * * *'  # Daily 01:00 UTC = 08:00 WIB
 7
 8jobs:
 9  cost-report:
10    runs-on: ubuntu-latest
11    env:
12      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
13    steps:
14      - uses: actions/checkout@v4
15      - uses: actions/setup-go@v5
16        with: { go-version: '1.22' }
17
18      - name: Fetch Anthropic usage
19        id: usage
20        run: |
21          # Anthropic API: usage data
22          USAGE=$(curl -sf \
23            "https://api.anthropic.com/v1/usage?start_date=$(date -d '1 day ago' +%Y-%m-%d)" \
24            -H "x-api-key: ${ANTHROPIC_API_KEY}" \
25            -H "anthropic-version: 2023-06-01")
26          echo "usage=$USAGE" >> $GITHUB_OUTPUT
27
28      - name: Generate AI cost report
29        run: |
30          USAGE='${{ steps.usage.outputs.usage }}'
31
32          PROMPT="Generate daily AI cost report for Santekno Shop engineering team.
33
34          Usage data: $USAGE
35
36          Format in Bahasa Indonesia:
37          📊 Daily AI Cost Report - $(date +%Y-%m-%d)
38
39          Total cost: [total]
40          By feature: [breakdown]
41          vs yesterday: [comparison]
42          Budget status: [% of monthly budget used]
43
44          Flag if any feature exceeded 80% of daily budget.
45          Max 150 words."
46
47          REPORT=$(./scripts/claude-call.sh "$PROMPT" 200)
48
49          curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \
50            -H "Content-Type: application/json" \
51            -d "$(jq -n --arg text "$REPORT" '{text: $text}')"

 1// internal/ai/cost/alert.go
 2
 3package cost
 4
 5import (
 6    "context"
 7    "encoding/json"
 8    "fmt"
 9    "net/http"
10    "strings"
11)
12
13type AlertManager struct {
14    slackWebhook string
15    tracker      *Tracker
16}
17
18func (a *AlertManager) CheckAndAlert(ctx context.Context, feature string) {
19    used, budget, exceeded := a.tracker.CheckBudget(feature)
20    if budget <= 0 { return }
21
22    pct := used / budget * 100
23
24    // Alert di 80% dan 100%
25    switch {
26    case exceeded:
27        a.sendAlert(ctx, fmt.Sprintf(
28            "🚨 *BUDGET EXCEEDED*: Feature `%s` has spent $%.4f (budget: $%.4f = %.0f%%)\n"+
29                "AI calls for this feature are now blocked.",
30            feature, used, budget, pct))
31    case pct >= 80:
32        a.sendAlert(ctx, fmt.Sprintf(
33            "⚠️ *Budget Warning*: Feature `%s` at %.0f%% of monthly budget "+
34                "($%.4f / $%.4f)",
35            feature, pct, used, budget))
36    }
37}
38
39func (a *AlertManager) sendAlert(ctx context.Context, message string) {
40    payload, _ := json.Marshal(map[string]string{"text": message})
41    req, _ := http.NewRequestWithContext(ctx, http.MethodPost,
42        a.slackWebhook, strings.NewReader(string(payload)))
43    req.Header.Set("Content-Type", "application/json")
44
45    client := &http.Client{}
46    resp, err := client.Do(req)
47    if err != nil {
48        fmt.Printf("alert failed: %v\n", err)
49        return
50    }
51    defer resp.Body.Close()
52}

 1// grafana-dashboard.json (simplified)
 2{
 3  "title": "AI/LLM Cost Dashboard",
 4  "panels": [
 5    {
 6      "title": "Daily Cost by Feature (USD)",
 7      "type": "timeseries",
 8      "targets": [{
 9        "expr": "increase(ai_cost_usd_total[1d])",
10        "legendFormat": "{{feature}}"
11      }]
12    },
13    {
14      "title": "Monthly Cost vs Budget",
15      "type": "gauge",
16      "targets": [{
17        "expr": "increase(ai_cost_usd_total[30d]) / ai_budget_usd * 100",
18        "legendFormat": "Budget Used %"
19      }]
20    },
21    {
22      "title": "Token Usage by Type",
23      "type": "timeseries",
24      "targets": [{
25        "expr": "rate(ai_tokens_total[1h])",
26        "legendFormat": "{{feature}} {{token_type}}"
27      }]
28    },
29    {
30      "title": "Cost per Request by Feature",
31      "type": "stat",
32      "targets": [{
33        "expr": "rate(ai_cost_usd_total[1h]) / rate(ai_requests_total[1h])",
34        "legendFormat": "{{feature}}"
35      }]
36    }
37  ]
38}

 1// internal/ai/cost/optimizer.go
 2// Reduce token usage tanpa mengorbankan quality
 3
 4package cost
 5
 6import (
 7    "strings"
 8    "unicode"
 9)
10
11// TrimPrompt mengurangi token count prompt
12func TrimPrompt(prompt string, maxLines int) string {
13    lines := strings.Split(prompt, "\n")
14    if len(lines) <= maxLines {
15        return prompt
16    }
17
18    trimmed := lines[:maxLines]
19    trimmed = append(trimmed, "[... truncated for cost optimization ...]")
20    return strings.Join(trimmed, "\n")
21}
22
23// ChunkDiff memecah large diff menjadi chunks
24func ChunkDiff(diff string, maxChunkLines int) []string {
25    lines := strings.Split(diff, "\n")
26    var chunks []string
27
28    for i := 0; i < len(lines); i += maxChunkLines {
29        end := i + maxChunkLines
30        if end > len(lines) { end = len(lines) }
31        chunks = append(chunks, strings.Join(lines[i:end], "\n"))
32    }
33
34    return chunks
35}
36
37// EstimateTokens rough estimate: 1 token ≈ 4 characters
38func EstimateTokens(text string) int {
39    // Count characters excluding whitespace
40    chars := 0
41    for _, r := range text {
42        if !unicode.IsSpace(r) { chars++ }
43    }
44    return chars/3 + len(strings.Fields(text))/2
45}

 1// internal/ai/cost/anomaly.go
 2// Deteksi cost spike yang tidak normal
 3
 4package cost
 5
 6import "math"
 7
 8type AnomalyDetector struct {
 9    // Rolling window of daily costs per feature
10    history map[string][]float64
11    windowSize int  // days
12}
13
14func (d *AnomalyDetector) IsAnomaly(feature string, todayCost float64) bool {
15    history := d.history[feature]
16    if len(history) < 3 {
17        return false  // not enough data
18    }
19
20    // Calculate mean and stddev of recent history
21    sum := 0.0
22    for _, c := range history {
23        sum += c
24    }
25    mean := sum / float64(len(history))
26
27    variance := 0.0
28    for _, c := range history {
29        variance += math.Pow(c-mean, 2)
30    }
31    stddev := math.Sqrt(variance / float64(len(history)))
32
33    // Anomaly if today > mean + 3*stddev (3-sigma rule)
34    threshold := mean + 3*stddev
35    return todayCost > threshold && todayCost > mean*2  // at least 2x normal
36}

 1// Attribute LLM cost ke PR dan commit untuk accountability
 2
 3type CostEvent struct {
 4    Feature     string
 5    CostUSD     float64
 6    InputTokens int64
 7    OutputTokens int64
 8    Model       string
 9
10    // Attribution
11    PRURL       string
12    CommitSHA   string
13    Author      string
14    Branch      string
15    Timestamp   time.Time
16}
17
18// Di CI pipeline:
19func recordPRCost(ctx context.Context, tracker *Tracker, usage Usage) {
20    event := CostEvent{
21        Feature:      usage.Feature,
22        Model:        usage.Model,
23        InputTokens:  usage.InputTokens,
24        OutputTokens: usage.OutputTokens,
25
26        // From GitHub environment
27        PRURL:     os.Getenv("GITHUB_SERVER_URL") + "/" +
28                   os.Getenv("GITHUB_REPOSITORY") + "/pull/" +
29                   os.Getenv("PR_NUMBER"),
30        CommitSHA: os.Getenv("GITHUB_SHA"),
31        Author:    os.Getenv("GITHUB_ACTOR"),
32        Branch:    os.Getenv("GITHUB_HEAD_REF"),
33        Timestamp: time.Now(),
34    }
35
36    // Store untuk monthly attribution report
37    saveEvent(ctx, event)
38}

 1#!/bin/bash
 2# scripts/monthly-cost-review.sh
 3# AI-powered analysis of monthly LLM costs
 4
 5MONTH=$(date +%Y-%m)
 6
 7# Collect cost data dari Prometheus
 8COST_DATA=$(curl -sf "https://metrics.santekno.com/api/v1/query" \
 9  --data-urlencode "query=increase(ai_cost_usd_total[30d])")
10
11TOKEN_DATA=$(curl -sf "https://metrics.santekno.com/api/v1/query" \
12  --data-urlencode "query=increase(ai_tokens_total[30d])")
13
14./scripts/claude-call.sh "Generate monthly LLM cost optimization report.
15
16Month: $MONTH
17Cost data by feature: $COST_DATA
18Token data by feature: $TOKEN_DATA
19
20Analyze:
211. Which features have highest cost? Worth the value?
222. Which features have unexpected cost spikes?
233. Token efficiency: output/input ratio - is it healthy?
244. Recommendations untuk reduce cost by 20% tanpa quality impact
25
26Format: executive summary + action items (Bahasa Indonesia, max 250 words)." 350

 1// internal/ai/cost/cache.go
 2// Cache AI responses untuk identical inputs
 3
 4package cost
 5
 6import (
 7    "crypto/sha256"
 8    "encoding/hex"
 9    "sync"
10    "time"
11)
12
13type ResponseCache struct {
14    mu      sync.RWMutex
15    entries map[string]*CacheEntry
16    ttl     time.Duration
17}
18
19type CacheEntry struct {
20    Response  string
21    CreatedAt time.Time
22    HitCount  int
23    CostSaved float64  // accumulated cost saved by this cache entry
24}
25
26func (c *ResponseCache) Key(model, prompt string) string {
27    h := sha256.Sum256([]byte(model + ":" + prompt))
28    return hex.EncodeToString(h[:])
29}
30
31func (c *ResponseCache) Get(key string) (string, bool) {
32    c.mu.RLock()
33    defer c.mu.RUnlock()
34
35    entry, ok := c.entries[key]
36    if !ok || time.Since(entry.CreatedAt) > c.ttl {
37        return "", false
38    }
39
40    entry.HitCount++
41    return entry.Response, true
42}
43
44func (c *ResponseCache) Set(key, response string, costSaved float64) {
45    c.mu.Lock()
46    defer c.mu.Unlock()
47
48    c.entries[key] = &CacheEntry{
49        Response:  response,
50        CreatedAt: time.Now(),
51        CostSaved: costSaved,
52    }
53}
54
55// Semantic cache: similar prompts return same response
56// For PR descriptions: same diff = same description
57// Cache key: hash(diff_content) not hash(full_prompt)

 1Cost comparison untuk PR review task (1500 input + 400 output tokens):
 2
 3claude-haiku-4-5-20251001:
 4  Cost: (1500 × $0.80 + 400 × $4.00) / 1,000,000 = $0.0028/request
 5  100 PRs/month: $0.28/month
 6
 7claude-sonnet-4-6:
 8  Cost: (1500 × $3.00 + 400 × $15.00) / 1,000,000 = $0.0105/request
 9  100 PRs/month: $1.05/month
10
11claude-opus-4-6:
12  Cost: (1500 × $15.00 + 400 × $75.00) / 1,000,000 = $0.0525/request
13  100 PRs/month: $5.25/month
14
15Decision framework:
16  Mechanical checks (error handling, types): haiku ← 3.75× cheaper than sonnet
17  PR description generation: haiku
18  Complex architecture review: sonnet (worth the quality premium)
19  One-off complex analysis: sonnet/opus
20
21Rule: use haiku for ALL automation in CI/CD.
22      Reserve sonnet/opus for interactive or high-stakes analysis.

 1// Track total cost per PR di CI pipeline
 2
 3type PRCostSummary struct {
 4    PRURL          string
 5    TotalCostUSD   float64
 6    ByStep         map[string]float64
 7    TokensUsed     map[string]int64
 8}
 9
10func (s *PRCostSummary) AddStep(step string, usage Usage, costUSD float64) {
11    s.ByStep[step] = costUSD
12    s.TokensUsed[step] = usage.InputTokens + usage.OutputTokens
13    s.TotalCostUSD += costUSD
14}
15
16func (s *PRCostSummary) Format() string {
17    return fmt.Sprintf(
18        "AI costs for this PR:\n"+
19        "  PR Review:     $%.4f\n"+
20        "  PR Description: $%.4f\n"+
21        "  Spec Audit:    $%.4f\n"+
22        "  TOTAL:         $%.4f",
23        s.ByStep["pr-review"],
24        s.ByStep["pr-description"],
25        s.ByStep["spec-audit"],
26        s.TotalCostUSD,
27    )
28}
29// Typical total: $0.007 per PR
30// Monthly (96 PRs): $0.67

 1// Periodic budget reset (monthly)
 2
 3func (t *Tracker) ResetMonthlyBudgets() {
 4    t.mu.Lock()
 5    defer t.mu.Unlock()
 6
 7    // Archive current month before reset
 8    for feature, cost := range t.monthly {
 9        t.archiveMonthlyCost(feature, cost, time.Now())
10    }
11
12    // Reset
13    t.monthly = make(map[string]float64)
14    t.lastReset = time.Now()
15
16    fmt.Println("Monthly AI budgets reset")
17}
18
19// Scheduled via cron atau background goroutine:
20func (t *Tracker) RunMonthlyReset(ctx context.Context) {
21    for {
22        now := time.Now()
23        // Next reset: 1st of next month at 00:00 UTC
24        next := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.UTC)
25
26        select {
27        case <-time.After(time.Until(next)):
28            t.ResetMonthlyBudgets()
29        case <-ctx.Done():
30            return
31        }
32    }
33}

 1Cost governance untuk fitur AI baru:
 2
 3< $5/bulan: Engineering team lead approve
 4$5 - $20/bulan: Engineering manager approve
 5$20 - $100/bulan: CTO approve
 6> $100/bulan: Board approval (cost center impact)
 7
 8Implementation:
 9  1. New AI feature PR harus ada cost estimate
10  2. Format: "Estimated AI cost: $X/month (Y requests/day × $Z/request)"
11  3. Reviewer check estimate reasonable
12  4. Budget config PR dibuat bersamaan
13
14Template untuk PR description:
15  ## AI Cost Estimate
16  - Feature: [feature name]
17  - Model: claude-haiku-4-5-20251001
18  - Avg tokens per request: [X] input + [Y] output
19  - Expected requests: [N]/day
20  - Daily cost: $[D]
21  - Monthly cost: $[M]
22  - Budget requested: $[B]/month

 1#!/bin/bash
 2# Fetch usage data langsung dari Anthropic console API
 3
 4ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY}"
 5START_DATE=$(date -d '7 days ago' +%Y-%m-%d)
 6END_DATE=$(date +%Y-%m-%d)
 7
 8# Note: Anthropic usage API endpoint (check latest docs)
 9USAGE=$(curl -sf \
10  "https://api.anthropic.com/v1/organizations/usage" \
11  -H "x-api-key: ${ANTHROPIC_API_KEY}" \
12  -H "anthropic-version: 2023-06-01" \
13  -d "{\"start_date\": \"${START_DATE}\", \"end_date\": \"${END_DATE}\"}")
14
15echo "=== Weekly Usage Report ==="
16echo "$USAGE" | jq '{
17    total_input_tokens: .usage.input_tokens,
18    total_output_tokens: .usage.output_tokens,
19    total_cost: .usage.total_cost
20}'

 1## Guidelines untuk Cost-Efficient Prompts (tambahkan ke CLAUDE.md)
 2
 3### Token Reduction Techniques
 4
 51. **Front-load instructions**: paling penting di awal
 6   - Model membaca linear, awal lebih penting
 7   - Kurangi repetisi di akhir prompt
 8
 92. **Limit input size**:
10   - Diff: max 300 baris (bukan seluruh file)
11   - Context: max 150 baris CLAUDE.md (bukan seluruh file)
12   - Spec: max 500 karakter per feature
13
143. **Structure output untuk brevity**:
15   - Minta JSON bukan prose untuk machine-readable output
16   - "CRITICAL: X" bukan "I noticed that there is a critical issue with X"
17
184. **Batch vs individual**:
19   - Review 5 file sekaligus lebih murah dari 5 individual calls
20   - Tapi: batch terlalu besar = quality drop
21
225. **Model routing**:
23   - Gunakan haiku untuk semua CI automation
24   - Sonnet hanya untuk satu-off complex analysis