{
  "__inputs": [
    {
      "name": "DS_CLICKHOUSE",
      "label": "ClickHouse",
      "description": "The ClickHouse datasource pointing at your abkit internal database (_ab_results).",
      "type": "datasource",
      "pluginId": "grafana-clickhouse-datasource",
      "pluginName": "ClickHouse"
    }
  ],
  "__requires": [
    { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" },
    { "type": "datasource", "id": "grafana-clickhouse-datasource", "name": "ClickHouse", "version": "1.0.0" }
  ],
  "title": "ab-analysis-kit — experiments",
  "uid": "abkit-experiments",
  "editable": true,
  "schemaVersion": 39,
  "version": 1,
  "timezone": "",
  "time": { "from": "now-90d", "to": "now" },
  "refresh": "",
  "tags": ["ab-analysis-kit", "experiments"],
  "templating": {
    "list": [
      {
        "name": "datasource",
        "label": "ClickHouse datasource",
        "type": "datasource",
        "query": "grafana-clickhouse-datasource",
        "current": {},
        "hide": 0
      },
      {
        "name": "experiment",
        "label": "Experiment",
        "type": "textbox",
        "query": "example_signup_test",
        "current": { "text": "example_signup_test", "value": "example_signup_test" },
        "hide": 0
      },
      {
        "name": "metric",
        "label": "Metric",
        "type": "textbox",
        "query": "example_signup_cr",
        "current": { "text": "example_signup_cr", "value": "example_signup_cr" },
        "hide": 0
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "stat",
      "title": "SRM-failing comparisons (should be 0)",
      "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
      "fieldConfig": {
        "defaults": {
          "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${datasource}" },
          "rawSql": "WITH latest AS (SELECT experiment, metric, name_1, name_2, argMax(srm_flag, end_ts) AS srm_flag FROM abkit_internal._ab_results FINAL GROUP BY experiment, metric, name_1, name_2) SELECT countIf(srm_flag) AS srm_failing_comparisons FROM latest",
          "format": "table"
        }
      ]
    },
    {
      "id": 2,
      "type": "table",
      "title": "Scoreboard — latest verdict per comparison",
      "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${datasource}" },
      "gridPos": { "h": 9, "w": 18, "x": 6, "y": 0 },
      "options": { "showHeader": true },
      "fieldConfig": { "defaults": {}, "overrides": [] },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${datasource}" },
          "rawSql": "WITH latest AS (SELECT experiment, metric, method_config_id, name_1, name_2, argMax(effect, end_ts) AS effect, argMax(left_bound, end_ts) AS lo, argMax(right_bound, end_ts) AS hi, argMax(pvalue, end_ts) AS pvalue, argMax(alpha, end_ts) AS alpha, argMax(reject, end_ts) AS reject, argMax(is_main_metric, end_ts) AS is_main_metric, argMax(srm_flag, end_ts) AS srm_flag, argMax(decision_blocked, end_ts) AS decision_blocked, argMax(insufficient_data, end_ts) AS insufficient_data, argMax(is_horizon, end_ts) AS is_horizon, argMax(ci_kind, end_ts) AS ci_kind, max(end_ts) AS as_of FROM abkit_internal._ab_results FINAL GROUP BY experiment, metric, method_config_id, name_1, name_2) SELECT metric, name_1 AS control, name_2 AS treatment, round(effect, 4) AS effect, round(lo, 4) AS ci_low, round(hi, 4) AS ci_high, round(pvalue, 4) AS pvalue, round(alpha, 4) AS alpha_eff, multiIf(decision_blocked OR srm_flag, 'BLOCKED (SRM)', insufficient_data, 'INSUFFICIENT', NOT (is_horizon OR ci_kind = 'always_valid'), 'PENDING', reject AND effect > 0, 'WIN', reject AND effect < 0, 'LOSE', NOT reject, 'FLAT', 'INCONCLUSIVE') AS verdict, as_of FROM latest WHERE experiment = '${experiment}' ORDER BY is_main_metric DESC, metric, control, treatment",
          "format": "table"
        }
      ]
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "Effect + CI stabilization — ${experiment} / ${metric}",
      "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${datasource}" },
      "gridPos": { "h": 10, "w": 24, "x": 0, "y": 9 },
      "options": { "tooltip": { "mode": "multi" }, "legend": { "displayMode": "list", "placement": "bottom" } },
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "fillOpacity": 10, "spanNulls": false },
          "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }
        },
        "overrides": [
          { "matcher": { "id": "byName", "options": "effect" }, "properties": [ { "id": "custom.lineWidth", "value": 2 } ] }
        ]
      },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${datasource}" },
          "rawSql": "SELECT end_ts AS t, effect, left_bound AS ci_low, right_bound AS ci_high FROM abkit_internal._ab_results FINAL WHERE experiment = '${experiment}' AND metric = '${metric}' AND effect IS NOT NULL ORDER BY method_config_id, name_1, name_2, end_ts",
          "format": "time_series"
        }
      ]
    },
    {
      "id": 4,
      "type": "table",
      "title": "Pipeline freshness (per experiment)",
      "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 19 },
      "options": { "showHeader": true },
      "fieldConfig": { "defaults": {}, "overrides": [] },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${datasource}" },
          "rawSql": "SELECT experiment, max(end_ts) AS latest_cutoff, max(created_at) AS last_computed_at, count(DISTINCT metric) AS metrics, count(DISTINCT method_config_id) AS method_configs FROM abkit_internal._ab_results FINAL GROUP BY experiment ORDER BY last_computed_at DESC",
          "format": "table"
        }
      ]
    }
  ]
}