Prompt Version Control Golang: Kelola Prompt seperti Source Code

 1Traditional DevOps:
 2  Code → Test → Deploy → Monitor → Iterate
 3
 4LLMOps (LLM Operations):
 5  Prompt → Evaluate → Deploy → Monitor → Iterate
 6
 7Tantangan LLM yang tidak ada di software biasa:
 8  1. Non-deterministic: output berbeda untuk input yang sama
 9  2. Quality drift: model update bisa mengubah output quality
10  3. Cost variability: prompt berbeda = cost yang sangat berbeda
11  4. No compilation error: prompt yang "salah" tidak error, hanya output buruk
12  5. Evaluation complexity: "benar" atau "salah" tidak selalu jelas

 1// internal/ai/prompt/registry.go
 2// Centralized prompt management
 3
 4package prompt
 5
 6import (
 7    "bytes"
 8    "fmt"
 9    "sync"
10    "text/template"
11)
12
13type Registry struct {
14    mu      sync.RWMutex
15    prompts map[string]*VersionedPrompt
16}
17
18type VersionedPrompt struct {
19    Name        string
20    Version     string
21    Template    *template.Template
22    Description string
23    Tags        []string // e.g., ["production", "v1.5"]
24}
25
26func NewRegistry() *Registry {
27    return &Registry{prompts: make(map[string]*VersionedPrompt)}
28}
29
30func (r *Registry) Register(name, version, tmplStr, description string) error {
31    tmpl, err := template.New(name).Parse(tmplStr)
32    if err != nil {
33        return fmt.Errorf("prompt.Register: %w", err)
34    }
35
36    r.mu.Lock()
37    defer r.mu.Unlock()
38
39    r.prompts[name] = &VersionedPrompt{
40        Name:        name,
41        Version:     version,
42        Template:    tmpl,
43        Description: description,
44    }
45    return nil
46}
47
48func (r *Registry) Render(name string, data any) (string, error) {
49    r.mu.RLock()
50    p, ok := r.prompts[name]
51    r.mu.RUnlock()
52
53    if !ok {
54        return "", fmt.Errorf("prompt %q not found", name)
55    }
56
57    var buf bytes.Buffer
58    if err := p.Template.Execute(&buf, data); err != nil {
59        return "", fmt.Errorf("prompt.Render %s: %w", name, err)
60    }
61    return buf.String(), nil
62}

 1internal/ai/prompts/
 2  order-review/
 3    v1.0.md       # production
 4    v1.1.md       # staging (being tested)
 5    v2.0-draft.md # development
 6  pr-description/
 7    v1.0.md
 8    v1.1.md       # current production
 9
10# Format prompt file (Markdown):
11---
12name: order-review
13version: 1.1
14description: Review order for cancel eligibility and spec compliance
15model: claude-haiku-4-5-20251001
16max_tokens: 500
17tags: [production, order]
18---
19
20You are reviewing an order cancellation request for Santekno Shop.
21
22Order details:
23- ID: {{.OrderID}}
24- Status: {{.Status}}
25- Age: {{.AgeMinutes}} minutes
26- Amount: Rp {{.Amount}}
27
28Review:
291. Is this order eligible for cancellation?
302. Are there any compliance concerns?
31
32Respond with: ELIGIBLE | NOT_ELIGIBLE | REQUIRES_REVIEW
33And brief reason.

 1// internal/ai/prompt/loader.go
 2// Load prompts dari filesystem
 3
 4package prompt
 5
 6import (
 7    "fmt"
 8    "os"
 9    "path/filepath"
10    "strings"
11
12    "gopkg.in/yaml.v3"
13)
14
15type PromptMeta struct {
16    Name        string   `yaml:"name"`
17    Version     string   `yaml:"version"`
18    Description string   `yaml:"description"`
19    Model       string   `yaml:"model"`
20    MaxTokens   int      `yaml:"max_tokens"`
21    Tags        []string `yaml:"tags"`
22}
23
24func LoadFromDirectory(dir string, registry *Registry) error {
25    return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
26        if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") {
27            return err
28        }
29
30        content, err := os.ReadFile(path)
31        if err != nil {
32            return fmt.Errorf("LoadFromDirectory: %w", err)
33        }
34
35        meta, body, err := parseFrontmatter(string(content))
36        if err != nil {
37            return fmt.Errorf("parse %s: %w", path, err)
38        }
39
40        return registry.Register(meta.Name, meta.Version, body, meta.Description)
41    })
42}
43
44func parseFrontmatter(content string) (*PromptMeta, string, error) {
45    parts := strings.SplitN(content, "---\n", 3)
46    if len(parts) < 3 {
47        return nil, content, nil // no frontmatter
48    }
49
50    var meta PromptMeta
51    if err := yaml.Unmarshal([]byte(parts[1]), &meta); err != nil {
52        return nil, "", fmt.Errorf("yaml unmarshal: %w", err)
53    }
54
55    return &meta, parts[2], nil
56}

 1Semantic versioning untuk prompts:
 2
 3v1.0.0: Initial production prompt
 4v1.0.1: Minor wording improvement (tidak mengubah behavior)
 5v1.1.0: New capability added (mengubah output format atau scope)
 6v2.0.0: Breaking change (output format berubah total)
 7
 8Git workflow untuk prompt changes:
 9  1. Edit prompt file di development branch
10  2. Run evaluation (lihat 15.07)
11  3. PR dengan hasil evaluation
12  4. Review: apakah quality naik atau turun?
13  5. Merge ke main → deploy ke production
14
15Rollback:
16  git revert <commit>
17  Atau: git checkout <ref> internal/ai/prompts/order-review/v1.0.md
18  Pipeline redeploy otomatis

 1// internal/ai/prompt/ab.go
 2// A/B testing untuk prompt comparison
 3
 4package prompt
 5
 6import "math/rand"
 7
 8type ABTest struct {
 9    PromptA      string  // name dari prompt A
10    PromptB      string  // name dari prompt B
11    TrafficSplit float64 // 0.0-1.0: fraction ke B
12}
13
14type ABRegistry struct {
15    *Registry
16    tests map[string]*ABTest
17}
18
19func (r *ABRegistry) RenderWithAB(testName string, data any) (string, string, error) {
20    test, ok := r.tests[testName]
21    if !ok {
22        // Fallback ke prompt default
23        result, err := r.Render(testName, data)
24        return result, testName, err
25    }
26
27    // Route traffic berdasarkan split
28    promptName := test.PromptA
29    if rand.Float64() < test.TrafficSplit {
30        promptName = test.PromptB
31    }
32
33    result, err := r.Render(promptName, data)
34    return result, promptName, err
35}

 1// cmd/eval-prompt/main.go
 2// Evaluate prompt quality sebelum deploy
 3
 4package main
 5
 6import (
 7    "encoding/json"
 8    "fmt"
 9    "os"
10)
11
12type TestCase struct {
13    Name     string `json:"name"`
14    Input    any    `json:"input"`
15    Expected string `json:"expected"` // ELIGIBLE | NOT_ELIGIBLE | REQUIRES_REVIEW
16}
17
18type EvalResult struct {
19    TestCase     string
20    Expected     string
21    Got          string
22    Pass         bool
23    LatencyMs    int64
24    InputTokens  int
25    OutputTokens int
26}
27
28func runEvaluation(promptName, testDataFile string) ([]EvalResult, error) {
29    data, err := os.ReadFile(testDataFile)
30    if err != nil {
31        return nil, fmt.Errorf("read test data: %w", err)
32    }
33
34    var testCases []TestCase
35    if err := json.Unmarshal(data, &testCases); err != nil {
36        return nil, fmt.Errorf("parse test cases: %w", err)
37    }
38
39    var results []EvalResult
40    for _, tc := range testCases {
41        results = append(results, runSingleCase(promptName, tc))
42    }
43    return results, nil
44}
45
46func main() {
47    promptName := os.Args[1]
48    testFile := os.Args[2]
49
50    results, err := runEvaluation(promptName, testFile)
51    if err != nil {
52        fmt.Fprintln(os.Stderr, err)
53        os.Exit(1)
54    }
55
56    passed := 0
57    for _, r := range results {
58        if r.Pass {
59            passed++
60        }
61        status := map[bool]string{true: "PASS", false: "FAIL"}[r.Pass]
62        fmt.Printf("[%s] %s: expected=%s got=%s\n", status, r.TestCase, r.Expected, r.Got)
63    }
64
65    passRate := float64(passed) / float64(len(results)) * 100
66    fmt.Printf("\nPass rate: %.1f%% (%d/%d)\n", passRate, passed, len(results))
67
68    if passRate < 80 {
69        fmt.Println("Evaluation FAILED (< 80% pass rate)")
70        os.Exit(1)
71    }
72    fmt.Println("Evaluation PASSED")
73}

 1// testdata/prompts/order-review-test-cases.json
 2[
 3  {
 4    "name": "PENDING order within window",
 5    "input": { "OrderID": "uuid-1", "Status": "PENDING", "AgeMinutes": 10, "Amount": 50000 },
 6    "expected": "ELIGIBLE"
 7  },
 8  {
 9    "name": "PENDING order expired window",
10    "input": { "OrderID": "uuid-2", "Status": "PENDING", "AgeMinutes": 45, "Amount": 75000 },
11    "expected": "NOT_ELIGIBLE"
12  },
13  {
14    "name": "CONFIRMED order",
15    "input": { "OrderID": "uuid-3", "Status": "CONFIRMED", "AgeMinutes": 5, "Amount": 25000 },
16    "expected": "NOT_ELIGIBLE"
17  },
18  {
19    "name": "Large order - extra review",
20    "input": { "OrderID": "uuid-4", "Status": "PENDING", "AgeMinutes": 20, "Amount": 5000000 },
21    "expected": "REQUIRES_REVIEW"
22  }
23]

 1# .github/workflows/prompt-eval.yml
 2name: Prompt Evaluation
 3
 4on:
 5  pull_request:
 6    paths:
 7      - 'internal/ai/prompts/**'
 8
 9jobs:
10  evaluate:
11    name: Evaluate Changed Prompts
12    runs-on: ubuntu-latest
13    env:
14      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
15    steps:
16      - uses: actions/checkout@v4
17      - uses: actions/setup-go@v5
18        with: { go-version: '1.22' }
19
20      - name: Find changed prompt files
21        id: changed
22        run: |
23          CHANGED=$(git diff origin/${{ github.base_ref }}...HEAD \
24            --name-only -- 'internal/ai/prompts/**')
25          echo "prompts=$CHANGED" >> $GITHUB_OUTPUT
26
27      - name: Build evaluator
28        run: go build -o /tmp/eval-prompt ./cmd/eval-prompt/
29
30      - name: Run evaluations
31        run: |
32          for prompt_file in ${{ steps.changed.outputs.prompts }}; do
33            PROMPT_NAME=$(grep "^name:" "$prompt_file" | awk '{print $2}')
34            TEST_FILE="testdata/prompts/${PROMPT_NAME}-test-cases.json"
35            if [ -f "$TEST_FILE" ]; then
36              echo "Evaluating: $PROMPT_NAME"
37              /tmp/eval-prompt "$PROMPT_NAME" "$TEST_FILE"
38            else
39              echo "::warning::No test cases for $PROMPT_NAME"
40            fi
41          done

 1// internal/ai/prompt/monitor.go
 2// Track prompt performance di production
 3
 4package prompt
 5
 6import "sync"
 7
 8type PromptMetrics struct {
 9    mu           sync.Mutex
10    Name         string
11    Version      string
12    Requests     int64
13    Errors       int64
14    AvgLatencyMs float64
15    InputTokens  int64
16    OutputTokens int64
17    QualityScore float64 // dari human feedback atau automated eval
18}
19
20func (m *PromptMetrics) Record(latencyMs int64, inputTokens, outputTokens int, err error) {
21    m.mu.Lock()
22    defer m.mu.Unlock()
23
24    m.Requests++
25    if err != nil {
26        m.Errors++
27    }
28
29    // Running average latency
30    m.AvgLatencyMs = (m.AvgLatencyMs*float64(m.Requests-1) + float64(latencyMs)) /
31        float64(m.Requests)
32
33    m.InputTokens += int64(inputTokens)
34    m.OutputTokens += int64(outputTokens)
35}
36
37func (m *PromptMetrics) ErrorRate() float64 {
38    if m.Requests == 0 {
39        return 0
40    }
41    return float64(m.Errors) / float64(m.Requests)
42}
43
44func (m *PromptMetrics) EstimatedCostUSD() float64 {
45    // Haiku pricing (per 1M tokens)
46    return (float64(m.InputTokens)*0.80 + float64(m.OutputTokens)*4.00) / 1_000_000
47}

 1// internal/ai/prompt/canary.go
 2// Canary deploy untuk prompt baru
 3
 4package prompt
 5
 6import (
 7    "math/rand"
 8    "sync/atomic"
 9)
10
11type CanaryConfig struct {
12    StableVersion string
13    CanaryVersion string
14    CanaryPercent float64 // 0.0-1.0
15}
16
17type CanaryRouter struct {
18    config *CanaryConfig
19    stats  struct {
20        stableRequests int64
21        canaryRequests int64
22        stableErrors   int64
23        canaryErrors   int64
24    }
25}
26
27func (r *CanaryRouter) Route() string {
28    if rand.Float64() < r.config.CanaryPercent {
29        atomic.AddInt64(&r.stats.canaryRequests, 1)
30        return r.config.CanaryVersion
31    }
32    atomic.AddInt64(&r.stats.stableRequests, 1)
33    return r.config.StableVersion
34}
35
36func (r *CanaryRouter) CanaryErrorRate() float64 {
37    reqs := atomic.LoadInt64(&r.stats.canaryRequests)
38    if reqs == 0 {
39        return 0
40    }
41    errs := atomic.LoadInt64(&r.stats.canaryErrors)
42    return float64(errs) / float64(reqs)
43}
44
45func (r *CanaryRouter) ShouldRollback() bool {
46    // Rollback jika canary error rate 2x lebih tinggi dari stable
47    stableReqs := atomic.LoadInt64(&r.stats.stableRequests)
48    stableErrs := atomic.LoadInt64(&r.stats.stableErrors)
49
50    if stableReqs == 0 {
51        return false
52    }
53    stableRate := float64(stableErrs) / float64(stableReqs)
54
55    return r.CanaryErrorRate() > stableRate*2.0
56}

 1// PromptManager dengan rollback capability
 2
 3type PromptManager struct {
 4    registry *Registry
 5    canary   *CanaryRouter
 6    metrics  map[string]*PromptMetrics
 7}
 8
 9func (m *PromptManager) GetPrompt(name string, data any) (string, error) {
10    // Route ke stable atau canary
11    version := m.canary.Route()
12
13    // Check apakah perlu rollback
14    if m.canary.ShouldRollback() {
15        log.Warn().Msg("Canary error rate too high — rolling back to stable")
16        m.canary.config.CanaryPercent = 0.0 // disable canary
17        version = m.canary.config.StableVersion
18        // Notify: "Prompt canary rolled back automatically"
19    }
20
21    return m.registry.Render(name+"-"+version, data)
22}

 1#!/bin/bash
 2# scripts/prompt-diff.sh
 3# Show meaningful diff antara dua prompt versions
 4
 5PROMPT_NAME="${1}"
 6OLD_VERSION="${2:-HEAD~1}"
 7NEW_VERSION="${3:-HEAD}"
 8
 9OLD=$(git show "${OLD_VERSION}:internal/ai/prompts/${PROMPT_NAME}.md" 2>/dev/null)
10NEW=$(git show "${NEW_VERSION}:internal/ai/prompts/${PROMPT_NAME}.md" 2>/dev/null)
11
12if [ -z "$OLD" ] || [ -z "$NEW" ]; then
13  echo "Prompt file not found in one of the versions"
14  exit 1
15fi
16
17diff <(echo "$OLD") <(echo "$NEW") --unified=3 --label "v_old" --label "v_new"
18
19# AI-powered diff summary:
20DIFF=$(diff <(echo "$OLD") <(echo "$NEW") --unified=3)
21./scripts/claude-call.sh "Summarize this prompt change in 2-3 sentences.
22What changed? What might change in AI behavior?
23Diff: $DIFF" 150

 1## AI Prompt Development Guidelines (tambahkan ke CLAUDE.md)
 2
 3### Prompt File Format
 4ALWAYS use YAML frontmatter:
 5  name, version, description, model, max_tokens, tags
 6
 7### Versioning Rules
 8Patch (x.y.Z): wording improvement, tidak ubah behavior
 9Minor (x.Y.0): tambah capability, output format compatible
10Major (X.0.0): breaking change di output format
11
12### Quality Requirements
13Setiap prompt harus ada test cases di testdata/prompts/
14Minimum pass rate: 80% di evaluation sebelum deploy
15
16### Template Variables
17Gunakan Go template syntax: {{.VariableName}}
18Pastikan semua variable ada di data struct yang dipass

 1// Prometheus metrics untuk prompt monitoring
 2
 3var (
 4    promptRequests = prometheus.NewCounterVec(
 5        prometheus.CounterOpts{Name: "ai_prompt_requests_total"},
 6        []string{"name", "version"},
 7    )
 8    promptLatency = prometheus.NewHistogramVec(
 9        prometheus.HistogramOpts{
10            Name:    "ai_prompt_duration_ms",
11            Buckets: []float64{100, 500, 1000, 2000, 5000},
12        },
13        []string{"name", "version"},
14    )
15    promptCostCents = prometheus.NewCounterVec(
16        prometheus.CounterOpts{Name: "ai_prompt_cost_cents_total"},
17        []string{"name", "version"},
18    )
19)
20
21func (m *PromptManager) callWithMetrics(name, version string, data any) (string, error) {
22    start := time.Now()
23    result, err := m.registry.Render(name+"-"+version, data)
24    latency := time.Since(start).Milliseconds()
25
26    promptRequests.WithLabelValues(name, version).Inc()
27    promptLatency.WithLabelValues(name, version).Observe(float64(latency))
28
29    return result, err
30}

 1#!/bin/bash
 2# scripts/prompt-security-review.sh
 3# Setiap prompt baru perlu security review
 4
 5PROMPT_CONTENT=$(cat "$1")
 6
 7./scripts/claude-call.sh "Review ini untuk prompt injection vulnerabilities:
 8
 9$PROMPT_CONTENT
10
11Check:
121. Apakah ada instruction yang bisa di-override oleh user input?
132. Apakah ada confidential system info yang ter-expose?
143. Apakah ada yang bisa menyebabkan AI ignore safety guidelines?
15
16Output: SECURE | VULNERABLE
17Detail jika VULNERABLE." 300

 1// internal/ai/prompt/model.go
 2// Support multiple LLM providers
 3
 4type ModelConfig struct {
 5    Provider  string // "anthropic", "openai", "gemini"
 6    ModelID   string
 7    MaxTokens int
 8}
 9
10var modelDefaults = map[string]ModelConfig{
11    "haiku":  {"anthropic", "claude-haiku-4-5-20251001", 1000},
12    "sonnet": {"anthropic", "claude-sonnet-4-6", 2000},
13    "gpt4o":  {"openai", "gpt-4o-mini", 1000},
14}
15
16func (r *Registry) RenderWithModel(name, modelAlias string, data any) (string, ModelConfig, error) {
17    model, ok := modelDefaults[modelAlias]
18    if !ok {
19        return "", ModelConfig{}, fmt.Errorf("unknown model alias: %s", modelAlias)
20    }
21
22    result, err := r.Render(name, data)
23    return result, model, err
24}

 1// internal/ai/prompt/budget.go
 2// Cost budget enforcement
 3
 4type BudgetManager struct {
 5    mu      sync.Mutex
 6    budgets map[string]float64 // prompt name → monthly budget USD
 7    spent   map[string]float64 // prompt name → spent this month
 8}
 9
10func (b *BudgetManager) Check(promptName string, estimatedCostUSD float64) error {
11    b.mu.Lock()
12    defer b.mu.Unlock()
13
14    budget, ok := b.budgets[promptName]
15    if !ok {
16        return nil // no budget set
17    }
18
19    if b.spent[promptName]+estimatedCostUSD > budget {
20        return fmt.Errorf("prompt %s: monthly budget $%.2f exceeded (spent: $%.2f)",
21            promptName, budget, b.spent[promptName])
22    }
23
24    return nil
25}

 1Prompt governance checklist untuk setiap prompt baru:
 2
 3Creation:
 4  [ ] Frontmatter lengkap (name, version, description, model, max_tokens)
 5  [ ] Template variables terdokumentasi
 6  [ ] Test cases ditambahkan (>= 5 per output category)
 7  [ ] Evaluation pass rate > 80%
 8  [ ] Security review selesai
 9  [ ] Estimasi cost dalam monthly budget
10
11Deployment:
12  [ ] Canary di 10% dulu
13  [ ] Monitor selama 24 jam
14  [ ] Promote ke 100% atau rollback berdasarkan metrics
15
16Updates:
17  [ ] Version dinaikkan sesuai aturan semantic
18  [ ] Test cases diupdate untuk behavior baru
19  [ ] Evaluation dijalankan ulang
20  [ ] PR direview oleh prompt author + satu engineer lain