Beanstalk Longshot test farm
Menu

#1725 WIP: perf(vk): late readbacks on by default on the Steam Deck and handhelds

by HumanBean closed Open on Forgejo

Commit

Head
fec5333
Base
main c4e8e2e
Approval
approved HumanBean, · reviewed

Plan (gpt-6.1-terra)

Compare default and forced late readbacks across three outdoor scenes

In plain words

The plan compares the PR with main on Steam Deck, Windows Intel, Linux Intel and an AMD integrated GPU; the Intel targets will wait for matching testers. It exercises standing and walking with camera movement in Kokiri Forest, Hyrule Field and the Market at an uncapped frame rate, measuring FPS and p99 frame time and comparing sampled pictures. It also compares forced newest and disabled readback rules on the PR; crashes, significant performance regressions or picture differences fail the plan. The requested frame log is unavailable to plans, so sampled pictures cannot reliably detect one-frame flashes. Each target takes roughly 35 minutes. No tester online now matches windows+intel, linux+intel: those targets wait for one.

Asked for

compare this PR with main on steamdeck (where the PR turns the rule on), and on windows+intel, linux+intel and an AMD integrated GPU with `RX_VK_READBACK_LATE=newest` forced against `RX_VK_READBACK_LATE=0` (whether integrated GPUs other than handhelds should get it): Kokiri Forest, Hyrule Field and the Market, standing and walking with the camera turning, the frame rate uncapped; fps, p99 frame time, the pictures against main, and the frame log (`RX_VK_FRAMELOG`) for one-frame white or dark pictures

Raw plan 413 lines Download plan.json

  "pass" 
    "crash" false
    "compare" 
      
        "axis" "RX_VK_READBACK_LATE"
        "side" "pr"
        "value" "newest"
        "metric" "frametime_p99_ms"
        "against" "0"
        "max_delta_pct" 15
      
      
        "axis" "RX_VK_READBACK_LATE"
        "side" "pr"
        "value" "newest"
        "metric" "fps_mean"
        "against" "0"
        "min_delta_pct" 10
      
    
    "fps_mean" 
      "min_delta_pct" 10
    
    "screenshot" 
      "max_diff" 0.05
    
    "frametime_p99_ms" 
      "max_delta_pct" 15
    
  
  "probe" 
    
      "kind" "crash"
    
    
      "kind" "frametime"
    
    
      "kind" "draws"
    
    
      "at_s" 
        5
      
      "kind" "screenshot"
      "upload" false
      "compare" "baseline"
    
  
  "matrix" 
    "RX_VK_READBACK_LATE" 
      "0"
      "newest"
    
  
  "schema" 8
  "target" 
    
      "name" "steamdeck"
      "tags" 
        "steamdeck"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "windows+intel"
      "tags" 
        "windows"
        "intel"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "linux+intel"
      "tags" 
        "linux"
        "intel"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "amd+igpu"
      "tags" 
        "amd"
        "igpu"
      
      "max_reports" 2
      "min_reports" 1
    
  
  "summary" "Compare default and forced late readbacks across three outdoor scenes"
  "baseline" 
    "ref" "main"
  
  "scenario" 
    
      "env" 
      "name" "default-policy"
      "runs" 1
      "step" 
        
          "do" "teleport"
          "to" "KokiriForest_Child"
        
        
          "do" "wait"
          "map" "KokiriForest_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "shot"
          "name" "forest"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "forest-standing"
          "warmup_s" 5
          "duration_s" 15
        
        
          "x" 0
          "y" 1
          "do" "stick"
          "stick" "L"
          "frames" 120
        
        
          "x" 0.7
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 120
        
        
          "do" "shot"
          "name" "forest-turned"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "forest-turned"
          "warmup_s" 0
          "duration_s" 10
        
        
          "do" "teleport"
          "to" "HyruleField_Child"
        
        
          "do" "wait"
          "map" "HyruleField_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "shot"
          "name" "field"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "field-standing"
          "warmup_s" 5
          "duration_s" 15
        
        
          "x" 0
          "y" 1
          "do" "stick"
          "stick" "L"
          "frames" 120
        
        
          "x" 0.7
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 120
        
        
          "do" "shot"
          "name" "field-turned"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "field-turned"
          "warmup_s" 0
          "duration_s" 10
        
        
          "do" "teleport"
          "to" "Market_Child"
        
        
          "do" "wait"
          "map" "Market_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "shot"
          "name" "market"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "market-standing"
          "warmup_s" 5
          "duration_s" 15
        
        
          "x" 0
          "y" 1
          "do" "stick"
          "stick" "L"
          "frames" 120
        
        
          "x" 0.7
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 120
        
        
          "do" "shot"
          "name" "market-turned"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "market-turned"
          "warmup_s" 0
          "duration_s" 10
        
      
      "start" "save:kokiri-start"
      "matrix" false
      "settings" 
        "frame_rate_cap" 6
      
    
    
      "env" 
      "name" "forced-rules"
      "runs" 1
      "step" 
        
          "do" "teleport"
          "to" "KokiriForest_Child"
        
        
          "do" "wait"
          "map" "KokiriForest_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "shot"
          "name" "forest"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "forest-standing"
          "warmup_s" 5
          "duration_s" 15
        
        
          "x" 0
          "y" 1
          "do" "stick"
          "stick" "L"
          "frames" 120
        
        
          "x" 0.7
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 120
        
        
          "do" "shot"
          "name" "forest-turned"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "forest-turned"
          "warmup_s" 0
          "duration_s" 10
        
        
          "do" "teleport"
          "to" "HyruleField_Child"
        
        
          "do" "wait"
          "map" "HyruleField_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "shot"
          "name" "field"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "field-standing"
          "warmup_s" 5
          "duration_s" 15
        
        
          "x" 0
          "y" 1
          "do" "stick"
          "stick" "L"
          "frames" 120
        
        
          "x" 0.7
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 120
        
        
          "do" "shot"
          "name" "field-turned"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "field-turned"
          "warmup_s" 0
          "duration_s" 10
        
        
          "do" "teleport"
          "to" "Market_Child"
        
        
          "do" "wait"
          "map" "Market_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "shot"
          "name" "market"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "market-standing"
          "warmup_s" 5
          "duration_s" 15
        
        
          "x" 0
          "y" 1
          "do" "stick"
          "stick" "L"
          "frames" 120
        
        
          "x" 0.7
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 120
        
        
          "do" "shot"
          "name" "market-turned"
          "upload" false
          "compare" "baseline"
        
        
          "do" "measure"
          "name" "market-turned"
          "warmup_s" 0
          "duration_s" 10
        
      
      "start" "save:kokiri-start"
      "matrix" true
      "settings" 
        "frame_rate_cap" 6
      
    
  

Targets

TargetTierReportsStatus
steamdeckreviewed HumanBean 1 1/1 fail
windows+intelreviewed HumanBean none yet 0/1 pass, 0 running, 0 queued
linux+intelreviewed HumanBean none yet 0/1 pass, 0 running, 0 queued
amd+igpureviewed HumanBean 1 2 1/1 fail

Reports

ReportCommitTargetTesterResultWhen
rp_962ec51d7be42244fec5333amd+igpu Hidden testererror
rp_fa69d489698e4331fec5333linux+intel Hidden testerfail off target
rp_96b6a50ed721a2b8fec5333steamdeck Flashman4/agent-0e17b1fail
rp_eeadeb2a0abe079bfec5333amd+igpu Flashman4/agent-0e17b1fail

Jobs

JobCommitTargetStateTesterStartedAttemptsEstimateNote
jb_e7b925a36c63ab8f logfec5333linux+intel off target done reportHidden tester1
jb_d254e7aafcdcd43d logfec5333amd+igpu done reportFlashman4/agent-0e17b11
jb_4f41e9123734b8b1 logfec5333steamdeck done reportFlashman4/agent-0e17b11
jb_57f7699b13dd9fff logfec5333linux+intel cancelled—0 the PR was closed
jb_49127d91ad020305 logfec5333windows+intel cancelled—0 the PR was closed
jb_ee0cf716a8a73255 logfec5333windows+intel cancelled—0 the PR was closed
jb_ee816a8ae5f7a439 logfec5333steamdeck cancelled—0 the PR was closed
jb_2fea7fe4fc875759 logfec5333amd+igpu done build-baseHidden tester1
jb_d9512fedd24f75ff logfec5333linux+intel cancelled—0 the PR was closed