Performance Regression Testing

// performance_regression_testing.kt — Demo benchmark dengan step counting (deterministik)

import kotlin.random.Random

// === Target function 1: BubbleSort dengan step counter ===
data class SortStats(var comparisons: Int = 0, var swaps: Int = 0)

fun bubbleSortCounted(arr: IntArray, s: SortStats) {
    val n = arr.size
    for (i in 0 until n - 1) {
        var swapped = false
        for (j in 0 until n - i - 1) {
            s.comparisons++
            if (arr[j] > arr[j + 1]) {
                val t = arr[j]; arr[j] = arr[j + 1]; arr[j + 1] = t
                s.swaps++
                swapped = true
            }
        }
        if (!swapped) return
    }
}

// === Target function 2: QuickSort dengan step counter ===
fun quickSortCounted(arr: IntArray, low: Int, high: Int, s: SortStats) {
    if (low < high) {
        val p = partitionCounted(arr, low, high, s)
        quickSortCounted(arr, low, p - 1, s)
        quickSortCounted(arr, p + 1, high, s)
    }
}

fun partitionCounted(arr: IntArray, low: Int, high: Int, s: SortStats): Int {
    val pivot = arr[high]
    var i = low - 1
    for (j in low until high) {
        s.comparisons++
        if (arr[j] <= pivot) {
            i++
            if (i != j) {
                val t = arr[i]; arr[i] = arr[j]; arr[j] = t
                s.swaps++
            }
        }
    }
    val t = arr[i + 1]; arr[i + 1] = arr[high]; arr[high] = t
    s.swaps++
    return i + 1
}

// === Target function 3: stdlib sortedArray (step = 1, dianggap black box) ===
fun stdlibSortCounted(arr: IntArray): IntArray = arr.sortedArray()

// === Benchmark runner: hitung step rata-rata, deterministic ===
data class BenchResult(
    val name: String,
    val iterations: Int,
    val totalComparisons: Long,
    val totalSwaps: Long,
    val meanComparisons: Double,
    val meanSwaps: Double
) {
    fun toRow(): String =
        "%-22s iter=%-4d meanCmp=%-9.1f meanSwap=%-9.1f".format(
            name, iterations, meanComparisons, meanSwaps
        )
}

fun benchmark(
    name: String,
    iterations: Int,
    seedBase: Int,
    input: (Int, Int) -> IntArray,
    action: (IntArray, SortStats) -> Unit
): BenchResult {
    val warmup = iterations / 5
    repeat(warmup) {
        val arr = input(seedBase + it, 200)
        action(arr, SortStats())
    }
    var totalCmp = 0L
    var totalSw = 0L
    val measured = iterations - warmup
    for (i in 0 until measured) {
        val arr = input(seedBase + warmup + i, 200)
        val s = SortStats()
        action(arr, s)
        totalCmp += s.comparisons
        totalSw += s.swaps
    }
    return BenchResult(
        name = name,
        iterations = measured,
        totalComparisons = totalCmp,
        totalSwaps = totalSw,
        meanComparisons = totalCmp.toDouble() / measured,
        meanSwaps = totalSw.toDouble() / measured
    )
}

fun makeInput(seed: Int, size: Int): IntArray {
    val rnd = Random(seed.toLong())
    return IntArray(size) { rnd.nextInt(0, 1000) }
}

fun main() {
    println("=== Performance Regression Testing ===")
    println()
    println("Benchmark dengan step counting (deterministik) untuk CI tracking.")
    println()

    val size = 200
    val iterations = 200
    val seedBase = 1000

    println("=== Konfigurasi ===")
    println("  Array size: $size elemen acak")
    println("  Iterasi per benchmark: $iterations (160 setelah warmup)")
    println("  Seed base: $seedBase (input identik antar run)")
    println()

    val results = listOf(
        benchmark("bubbleSort", iterations, seedBase, ::makeInput) { arr, s -> bubbleSortCounted(arr, s) },
        benchmark("quickSort", iterations, seedBase, ::makeInput) { arr, s -> quickSortCounted(arr, 0, arr.size - 1, s) }
    )

    println("=== Hasil Benchmark ===")
    results.forEach { println("  " + it.toRow()) }
    println()

    // === Deteksi regression ===
    println("=== Deteksi Regression ===")
    val baselineMean = results[0].meanComparisons
    results.forEach { r ->
        val ratio = baselineMean / r.meanComparisons
        val status = if (ratio > 1.0) "lebih lambat dari baseline" else "lebih cepat dari baseline"
        println("  %-22s rasio=%.2fx %s".format(r.name, ratio, status))
    }
    println()

    // === Format output JSON (untuk CI) ===
    println("=== Output JSON (untuk CI tracking) ===")
    println("{")
    println("  \"benchmark\": \"sort_step_counting\",")
    println("  \"size\": $size,")
    println("  \"results\": [")
    results.forEachIndexed { idx, r ->
        val trailing = if (idx < results.size - 1) "," else ""
        println("    {\"name\": \"${r.name}\", \"meanComparisons\": ${"%.1f".format(r.meanComparisons)}, \"iterations\": ${r.iterations}}$trailing")
    }
    println("  ]")
    println("}")
    println()

    // === Threshold regression ===
    println("=== Threshold Regression ===")
    println("  CI membandingkan meanComparisons dengan baseline pada commit sebelumnya")
    println("  Bila mean baru > 1.10x baseline -> WARNING (>10% regression)")
    println("  Bila mean baru > 1.25x baseline -> FAIL (>25% regression)")
    println()

    // === Aturan praktis ===
    println("=== Aturan Praktis ===")
    val rules = listOf(
        "1. Selalu lakukan warmup sebelum pengukuran (JIT perlu waktu)",
        "2. Pakai seed acak yang sama antar run (Random(seed))",
        "3. Bandingkan mean + percentile, bukan hanya min/max",
        "4. Track hasil di CI dengan JSON output ke timeseries DB",
        "5. Set threshold warning 10%, fail 25% dari baseline",
        "6. Jangan benchmark di mesin noise; pakai dedicated runner",
        "7. Untuk output deterministik, hitung langkah (step) bukan waktu"
    )
    rules.forEach { println("  " + it) }
}

=== Performance Regression Testing ===

Benchmark dengan step counting (deterministik) untuk CI tracking.

=== Konfigurasi ===
  Array size: 200 elemen acak
  Iterasi per benchmark: 200 (160 setelah warmup)
  Seed base: 1000 (input identik antar run)

=== Hasil Benchmark ===
  bubbleSort             iter=160  meanCmp=19749.0   meanSwap=9987.5   
  quickSort              iter=160  meanCmp=1556.9    meanSwap=715.3    

=== Deteksi Regression ===
  bubbleSort             rasio=1.00x lebih cepat dari baseline
  quickSort              rasio=12.68x lebih lambat dari baseline

=== Output JSON (untuk CI tracking) ===
{
  "benchmark": "sort_step_counting",
  "size": 200,
  "results": [
    {"name": "bubbleSort", "meanComparisons": 19749.0, "iterations": 160},
    {"name": "quickSort", "meanComparisons": 1556.9, "iterations": 160}
  ]
}

=== Threshold Regression ===
  CI membandingkan meanComparisons dengan baseline pada commit sebelumnya
  Bila mean baru > 1.10x baseline -> WARNING (>10% regression)
  Bila mean baru > 1.25x baseline -> FAIL (>25% regression)

=== Aturan Praktis ===
  1. Selalu lakukan warmup sebelum pengukuran (JIT perlu waktu)
  2. Pakai seed acak yang sama antar run (Random(seed))
  3. Bandingkan mean + percentile, bukan hanya min/max
  4. Track hasil di CI dengan JSON output ke timeseries DB
  5. Set threshold warning 10%, fail 25% dari baseline
  6. Jangan benchmark di mesin noise; pakai dedicated runner
  7. Untuk output deterministik, hitung langkah (step) bukan waktu

java -jar benchmarks.jar \
  -wi 5 -i 10 -f 1 \
  -rf json -rff benchmark-results.json