Beanstalk Longshot test farm
Menu

#1866 perf(vk): the recorder takes a whole part and submits it, so the render thread never waits at a part's end

by HumanBean closed Open on Forgejo

Commit

Head
f08644e
Base
main 89c3d21
Run
2
Approval
approved HumanBean, · reviewed

Plan (gpt-6.1-terra)

Compare recorder submission in three areas across requested platforms

In plain words

Against main, three starts per side will visit Kokiri Forest, Hyrule Field and Market with V-Sync off and the frame-rate cap uncapped; threaded rendering remains at its default Auto. Each area gets a 20-second standing measurement and a camera-turn exercise with comparison pictures; the farm cannot measure while issuing the turn input. The plan measures FPS, p99 frame time and stutters, and fails on crashes, area-arrival timeouts, substantial performance regressions or picture differences. Each target takes roughly 35 minutes; the Windows and Linux Intel targets will wait for matching testers to come online. No tester online now matches windows+intel, linux+intel: those targets wait for one.

Asked for

Please test this PR against main on every platform: windows+amd, windows+nvidia, windows+intel, linux+amd, linux+nvidia, linux+intel, the steamdeck, amd+igpu. Settings vsync=2 (Off) and frame_rate_cap=6 (Uncapped), Threaded rendering Auto. From the `kokiri-start` save: Kokiri Forest, Hyrule Field and Castle Town Market, in each 20 s standing then 15 s turning the camera, three game starts a side; make sure each side has reached the area before measuring it. Measure fps, p99 frame time, stutters, crashes and hangs (a hang is the main risk: this changes which thread submits the GPU's work), and compare pictures with main. This only acts where the recorder thread runs (6+ cores); on the Steam Deck and 4-core PCs it should be even.

Raw plan 258 lines Download plan.json

  "pass" 
    "crash" false
    "fps_mean" 
      "min_delta_pct" 5
    
    "screenshot" 
      "max_diff" 0.05
    
    "stutter_count" 
      "max_delta_pct" 20
    
    "frametime_p99_ms" 
      "max_delta_pct" 10
    
    "log_forbidden_hits" 
      "max" 0
    
  
  "probe" 
    
      "kind" "crash"
    
    
      "kind" "frametime"
    
    
      "kind" "draws"
    
    
      "kind" "log_match"
      "expected" 
      "forbidden" 
        "vk: device lost"
      
    
  
  "matrix" 
  "schema" 3
  "target" 
    
      "name" "windows+amd"
      "tags" 
        "windows"
        "amd"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "windows+nvidia"
      "tags" 
        "windows"
        "nvidia"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "windows+intel"
      "tags" 
        "windows"
        "intel"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "linux+amd"
      "tags" 
        "linux"
        "amd"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "linux+nvidia"
      "tags" 
        "linux"
        "nvidia"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "linux+intel"
      "tags" 
        "linux"
        "intel"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "steamdeck"
      "tags" 
        "steamdeck"
      
      "max_reports" 2
      "min_reports" 1
    
    
      "name" "amd+igpu"
      "tags" 
        "amd"
        "igpu"
      
      "max_reports" 2
      "min_reports" 1
    
  
  "summary" "Compare recorder submission in three areas across requested platforms"
  "baseline" 
    "ref" "main"
  
  "scenario" 
    
      "env" 
      "name" "three-areas"
      "runs" 3
      "step" 
        
          "do" "teleport"
          "to" "KokiriForest_Child"
        
        
          "do" "wait"
          "map" "KokiriForest_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "measure"
          "name" "forest-standing"
          "warmup_s" 5
          "duration_s" 20
        
        
          "do" "shot"
          "name" "forest-standing"
          "upload" false
          "compare" "baseline"
        
        
          "x" 1
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 600
        
        
          "x" 1
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 300
        
        
          "do" "shot"
          "name" "forest-turned"
          "upload" false
          "compare" "baseline"
        
        
          "do" "teleport"
          "to" "HyruleField_Child"
        
        
          "do" "wait"
          "map" "HyruleField_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "measure"
          "name" "field-standing"
          "warmup_s" 5
          "duration_s" 20
        
        
          "do" "shot"
          "name" "field-standing"
          "upload" false
          "compare" "baseline"
        
        
          "x" 1
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 600
        
        
          "x" 1
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 300
        
        
          "do" "shot"
          "name" "field-turned"
          "upload" false
          "compare" "baseline"
        
        
          "do" "teleport"
          "to" "Market_Child"
        
        
          "do" "wait"
          "map" "Market_Child"
          "event" "map"
          "timeout_s" 60
        
        
          "do" "measure"
          "name" "market-standing"
          "warmup_s" 5
          "duration_s" 20
        
        
          "do" "shot"
          "name" "market-standing"
          "upload" false
          "compare" "baseline"
        
        
          "x" 1
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 600
        
        
          "x" 1
          "y" 0
          "do" "stick"
          "stick" "R"
          "frames" 300
        
        
          "do" "shot"
          "name" "market-turned"
          "upload" false
          "compare" "baseline"
        
      
      "start" "save:kokiri-start"
      "matrix" true
      "settings" 
        "vsync" 2
        "frame_rate_cap" 6
      
    
  

Targets

TargetTierReportsStatus
windows+amdreviewed HumanBean 1 1/1 fail
windows+nvidiareviewed HumanBean 1 0/1 pass, 1 error
windows+intelreviewed HumanBean none yet 0/1 pass, 0 running, 0 queued
linux+amdreviewed HumanBean none yet 0/1 pass, 0 running, 0 queued
linux+nvidiareviewed HumanBean 1 2 1/1 fail
linux+intelreviewed HumanBean none yet 0/1 pass, 0 running, 0 queued
steamdeckreviewed HumanBean none yet 0/1 pass, 0 running, 0 queued
amd+igpureviewed HumanBean none yet 0/1 pass, 0 running, 0 queued

Reports

ReportCommitTargetTesterResultWhen
rp_40dabb02ecc42fd9f08644e run 2linux+nvidia Hidden testererror
rp_58e348c5ebac64fff08644e run 2linux+nvidia Nerdtality/agent-0452e3fail outlier
rp_c89210d0846b746ff08644e run 2windows+amd Razytos/agent-d8f902fail outlier
rp_9993b02c2c799c3bf08644e run 2windows+nvidia CubanLegend/agent-341924error
rp_b5cf890ddf8fdfeaf08644esteamdeck Flashman4/agent-0e17b1fail
rp_f47d73c8897e8c71f08644eamd+igpu Flashman4/agent-0e17b1fail
rp_a469be1b399ffdbdf08644elinux+nvidia Nerdtality/agent-0452e3fail outlier
rp_69e22fd96ba9a4f5f08644elinux+amd icyblue8/agent-368f00fail
rp_d549463ccce0ea2ff08644elinux+nvidia jabbaxipil/Blainefail outlier
rp_28cd9c2a5d5a15aaf08644ewindows+amd Hidden testerfail outlier
rp_24abd5b090f1f8b3f08644ewindows+nvidia Samsheperd/agent-a9f512fail
rp_18b65d3e2212c214f08644ewindows+amd Razytos/agent-d8f902fail outlier
rp_ff5f883c9bb9d420f08644ewindows+nvidia CubanLegend/agent-341924error

Jobs

JobCommitTargetStateTesterStartedAttemptsEstimateNote
jb_e19ead19ca880bd4 logf08644elinux+intel cancelled—0 the PR was closed
jb_059387f2c9d57b40 logf08644elinux+amd cancelled—3 the PR was closed
jb_e5ed3bbf69e5dfda logf08644ewindows+amd cancelled runHidden tester1 the PR was closed
jb_e9cd4f911cbdb97a logf08644ewindows+intel cancelled—0 the PR was closed
jb_8b085a4f7524c2b0 logf08644ewindows+amd done runRazytos/agent-d8f9021
jb_36ad5aa96783e3dc logf08644ewindows+nvidia done build-baseCubanLegend/agent-3419241
jb_8c7ec8028531a691 logf08644elinux+nvidia done runNerdtality/agent-0452e31
jb_13270176ad890acb logf08644elinux+nvidia done runHidden tester1
jb_09a77b3aeefabf34 logf08644elinux+amd cancelled—3 the PR was closed
jb_3647920838babe3d logf08644ewindows+intel cancelled—0 the PR was closed
jb_fdf6fff16ddab8f6 logf08644eamd+igpu cancelled—1 the PR was closed
jb_f2ee8a9fe860e845 logf08644elinux+intel cancelled—0 the PR was closed
jb_2571797ddcaf0ae0 logf08644esteamdeck cancelled—0 the PR was closed
jb_c6aa7dfd411f0fc8 logf08644esteamdeck cancelled—0 the PR was closed
jb_ff2aea9ecfa04a2f logf08644ewindows+nvidia cancelled runthebasedpisces/agent-8d29991 the PR was closed
jb_c65f173a02bd2dfe logf08644eamd+igpu cancelled—0 the PR was closed
jb_6c6ef3d176cfe738 logf08644elinux+amd off target cancelled runHidden tester1 report: the farm refused the report: the coordinator answered HTTP 400 (BadRequest): {"error":"report.rules[8].status must be one of pass, fail, unsupported, skipped, mismatch"}
jb_014b294408e11d0c logf08644eamd+igpu cancelled—0 re-planned by HumanBean
jb_59058eddd19e2a77 logf08644ewindows+intel cancelled—0 re-planned by HumanBean
jb_b9c09ca42dba084a logf08644elinux+intel cancelled—0 re-planned by HumanBean
jb_cd4badfecf8790c0 logf08644esteamdeck cancelled—0 re-planned by HumanBean
jb_8d64ca719d102eca logf08644ewindows+nvidia done build-baseCubanLegend/agent-3419241
jb_5ad747b00df33c6a logf08644elinux+nvidia done runNerdtality/agent-0452e32 report: the farm refused the report: the coordinator answered HTTP 400 (BadRequest): {"error":"report.rules[8].status must be one of pass, fail, unsupported, skipped, mismatch"}
jb_d7f757f4ab0fcfc7 logf08644ewindows+intel cancelled—1 re-planned by HumanBean
jb_0f9640f09742f6ae logf08644elinux+amd done reporticyblue8/agent-368f001
jb_5d02fa36adbc1897 logf08644ewindows+nvidia done runSamsheperd/agent-a9f5121
jb_ed72347a7a5ba3fc logf08644esteamdeck done reportFlashman4/agent-0e17b12 stopped by the tester: you stopped testing
jb_a37031b0eae8c83e logf08644elinux+nvidia done runjabbaxipil/Blaine1
jb_96d2e03480909ed6 logf08644ewindows+amd done runHidden tester1
jb_51ee259a83a4129a logf08644elinux+amd cancelled prefetchFlashman4/agent-0e17b14 re-planned by HumanBean
jb_377c393f48c2c691 logf08644eamd+igpu done reportFlashman4/agent-0e17b11
jb_d83388282c2f5fd2 logf08644ewindows+amd done reportRazytos/agent-d8f9021
jb_b3cc2de5a8f1c764 logf08644elinux+intel cancelled—0 re-planned by HumanBean