examples/shellbench.yaml

api_version: harbor-hf/v1alpha1
kind: Experiment
metadata:
  name: shellbench-qwen-hardware
  labels:
    purpose: serving-comparison
benchmark:
  dataset: harbor/shellbench@1.0
  dataset_digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
  task_names:
    - "*"
  task_digests:
    replace-with-resolved-task-name: sha256:0000000000000000000000000000000000000000000000000000000000000000
matrix:
  models:
    - id: qwen36-nvfp4
      repo: nvidia/Qwen3.6-35B-A3B-NVFP4
      revision: 0123456789abcdef0123456789abcdef01234567
      weights:
        format: safetensors
        quantization:
          method: modelopt
          scheme: nvfp4
  deployments:
    - id: rtx-pro-6000
      hardware: rtx-pro-6000
      accelerator_count: 1
      region: aws-us-east-1
      engine:
        name: vllm
        image: ghcr.io/example/vllm@sha256:0000000000000000000000000000000000000000000000000000000000000000
        arguments:
          - --model
          - /repository
          - --max-model-len
          - "65536"
          - --kv-cache-dtype
          - fp8
        environment:
          VLLM_USE_FLASHINFER_MOE_FP4: "1"
        secret_names:
          - HF_TOKEN
      parameters:
        min_replicas: 0
        max_replicas: 1
    - id: h200
      hardware: h200
      accelerator_count: 1
      region: aws-us-east-1
      engine:
        name: vllm
        image: ghcr.io/example/vllm@sha256:0000000000000000000000000000000000000000000000000000000000000000
        arguments:
          - --model
          - /repository
          - --max-model-len
          - "65536"
          - --kv-cache-dtype
          - fp8
        secret_names:
          - HF_TOKEN
  agents:
    - id: openclaw
      name: openclaw
      revision: 2026.7.2
      revision_kind: package
      parameters:
        thinking: "off"
        compaction: true
execution:
  attempts: 1
  concurrent_trials: 8
  timeout_seconds: 7200
artifacts:
  bucket: example/benchmark-runs
  trial_evidence:
    workspace_root: /app
    workspace_max_nodes: 100000
    workspace_max_file_bytes: 536870912
    workspace_max_total_bytes: 2147483648
    workspace_max_archive_bytes: 2147483648
    workspace_capture_timeout_seconds: 300
    judge_max_request_bytes: 33554432
    judge_max_response_bytes: 33554432
    judge_timeout_seconds: 300
    judge_max_calls_per_execution: 4
publishing:
  dataset: example/shellbench-results
  dataset_visibility: public
  index_dataset: example/benchmark-run-index
  index_dataset_visibility: public
  evaluation_id: shellbench-qwen-hardware
  role: final