Skip to content

Commit b5cf431

Browse files
NonaryCopilot
andcommitted
fix(video): eliminate null chosen_encoder crash during concurrent probe
When a serverinfo HTTP request triggers probe_encoders() while a streaming session is active, the function sets chosen_encoder = nullptr at the start of the re-probe. A concurrent capture thread reading chosen_encoder without synchronization dereferences the null pointer and crashes (ACCESS_VIOLATION at encoder_t::flags offset 0x240). - Replace chosen_encoder with a local new_encoder variable for the entire probe selection process, so the global pointer is never transiently null during probing - Publish chosen_encoder = new_encoder only after the probe fully succeeds; on failure the previous encoder is preserved - Add defensive null checks at all unsynchronized read sites (capture, encode_run_sync, capture_async) that snapshot the pointer before dereferencing Generated with [GitHub Copilot](https://github.com/features/copilot) Model: Claude Opus 4.6 High Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 024dd51 commit b5cf431

File tree

1 file changed

+36
-15
lines changed

1 file changed

+36
-15
lines changed

src/video.cpp

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2484,7 +2484,12 @@ namespace video {
24842484
std::vector<std::string> &display_names,
24852485
int &display_p
24862486
) {
2487-
const auto &encoder = *chosen_encoder;
2487+
const auto *enc_ptr = chosen_encoder;
2488+
if (!enc_ptr) {
2489+
BOOST_LOG(error) << "No encoder available for sync encoding"sv;
2490+
return encode_e::error;
2491+
}
2492+
const auto &encoder = *enc_ptr;
24882493

24892494
std::shared_ptr<platf::display_t> disp;
24902495

@@ -2726,7 +2731,12 @@ namespace video {
27262731
display = ref->display_wp->lock();
27272732
}
27282733

2729-
auto &encoder = *chosen_encoder;
2734+
auto *enc_ptr = chosen_encoder;
2735+
if (!enc_ptr) {
2736+
BOOST_LOG(error) << "No encoder available for async capture"sv;
2737+
return;
2738+
}
2739+
auto &encoder = *enc_ptr;
27302740

27312741
auto encode_device = make_encode_device(*display, encoder, config);
27322742
if (!encode_device) {
@@ -2766,10 +2776,17 @@ namespace video {
27662776
config_t config,
27672777
void *channel_data
27682778
) {
2779+
// Snapshot the encoder pointer to avoid races with concurrent probe_encoders() calls
2780+
auto *encoder = chosen_encoder;
2781+
if (!encoder) {
2782+
BOOST_LOG(error) << "No encoder available for capture"sv;
2783+
return;
2784+
}
2785+
27692786
auto idr_events = mail->event<bool>(mail::idr);
27702787

27712788
idr_events->raise(true);
2772-
if (chosen_encoder->flags & PARALLEL_ENCODING) {
2789+
if (encoder->flags & PARALLEL_ENCODING) {
27732790
capture_async(std::move(mail), config, channel_data);
27742791
} else {
27752792
safe::signal_t join_event;
@@ -3106,6 +3123,7 @@ namespace video {
31063123
const auto previous_active_av1_mode = active_av1_mode;
31073124
const auto previous_last_ref_frames_invalidation = last_encoder_probe_supported_ref_frames_invalidation;
31083125
const auto previous_last_yuv444_for_codec = last_encoder_probe_supported_yuv444_for_codec;
3126+
auto previous_encoder = chosen_encoder;
31093127

31103128
auto restore_previous_probe_state = util::fail_guard([&]() {
31113129
active_hevc_mode = previous_active_hevc_mode;
@@ -3116,9 +3134,9 @@ namespace video {
31163134

31173135
auto encoder_list = encoders;
31183136

3119-
// Restart encoder selection
3120-
auto previous_encoder = chosen_encoder;
3121-
chosen_encoder = nullptr;
3137+
// Use a local variable for encoder selection during probing so that
3138+
// chosen_encoder is never null while concurrent capture threads may read it.
3139+
encoder_t *new_encoder = nullptr;
31223140
active_hevc_mode = config::video.hevc_mode;
31233141
active_av1_mode = config::video.av1_mode;
31243142
last_encoder_probe_supported_ref_frames_invalidation = false;
@@ -3161,22 +3179,22 @@ namespace video {
31613179
// We will return an encoder here even if it fails one of the codec requirements specified by the user
31623180
adjust_encoder_constraints(encoder);
31633181

3164-
chosen_encoder = encoder;
3182+
new_encoder = encoder;
31653183
break;
31663184
}
31673185

31683186
pos++;
31693187
});
31703188

3171-
if (chosen_encoder == nullptr) {
3189+
if (new_encoder == nullptr) {
31723190
BOOST_LOG(error) << "Couldn't find any working encoder matching ["sv << config::video.encoder << ']';
31733191
}
31743192
}
31753193

31763194
BOOST_LOG(info) << "// Testing for available encoders, this may generate errors. You can safely ignore those errors. //"sv;
31773195

31783196
// If we haven't found an encoder yet, but we want one with specific codec support, search for that now.
3179-
if (chosen_encoder == nullptr && (active_hevc_mode >= 2 || active_av1_mode >= 2)) {
3197+
if (new_encoder == nullptr && (active_hevc_mode >= 2 || active_av1_mode >= 2)) {
31803198
KITTY_WHILE_LOOP(auto pos = std::begin(encoder_list), pos != std::end(encoder_list), {
31813199
auto encoder = *pos;
31823200

@@ -3200,18 +3218,18 @@ namespace video {
32003218
continue;
32013219
}
32023220

3203-
chosen_encoder = encoder;
3221+
new_encoder = encoder;
32043222
break;
32053223
});
32063224

3207-
if (chosen_encoder == nullptr) {
3225+
if (new_encoder == nullptr) {
32083226
BOOST_LOG(error) << "Couldn't find any working encoder that meets HEVC/AV1 requirements"sv;
32093227
}
32103228
}
32113229

32123230
// If no encoder was specified or the specified encoder was unusable, keep trying
32133231
// the remaining encoders until we find one that passes validation.
3214-
if (chosen_encoder == nullptr) {
3232+
if (new_encoder == nullptr) {
32153233
KITTY_WHILE_LOOP(auto pos = std::begin(encoder_list), pos != std::end(encoder_list), {
32163234
auto encoder = *pos;
32173235

@@ -3226,12 +3244,12 @@ namespace video {
32263244
// We will return an encoder here even if it fails one of the codec requirements specified by the user
32273245
adjust_encoder_constraints(encoder);
32283246

3229-
chosen_encoder = encoder;
3247+
new_encoder = encoder;
32303248
break;
32313249
});
32323250
}
32333251

3234-
if (chosen_encoder == nullptr) {
3252+
if (new_encoder == nullptr) {
32353253
const auto output_name = display_device::map_output_name(config::get_active_output_name());
32363254
BOOST_LOG(fatal) << "Unable to find display or encoder during startup."sv;
32373255
if (!config::video.adapter_name.empty() || !output_name.empty()) {
@@ -3247,7 +3265,7 @@ namespace video {
32473265
BOOST_LOG(info) << "// Ignore any errors mentioned above, they are not relevant. //"sv;
32483266
BOOST_LOG(info);
32493267

3250-
auto &encoder = *chosen_encoder;
3268+
auto &encoder = *new_encoder;
32513269

32523270
last_encoder_probe_supported_ref_frames_invalidation = (encoder.flags & REF_FRAMES_INVALIDATION);
32533271
last_encoder_probe_supported_yuv444_for_codec[0] = encoder.h264[encoder_t::PASSED] &&
@@ -3307,6 +3325,9 @@ namespace video {
33073325
hevc_hdr_supported,
33083326
av1_passed,
33093327
av1_hdr_supported);
3328+
// Publish the new encoder only after the probe has fully succeeded,
3329+
// so concurrent capture threads never observe a null chosen_encoder.
3330+
chosen_encoder = new_encoder;
33103331
restore_previous_probe_state.disable();
33113332
return 0;
33123333
}

0 commit comments

Comments
 (0)