|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +#version 450 core |
| 10 | + |
| 11 | +#define PRECISION ${PRECISION} |
| 12 | +#define VEC4_T ${texel_load_type(DTYPE, STORAGE)} |
| 13 | +#define T ${texel_load_component_type(DTYPE, STORAGE)} |
| 14 | + |
| 15 | +#define NUM_OUTPUTS_PER_WG ${NUM_OUTPUTS_PER_WG} |
| 16 | +#define NUM_WORKERS_PER_OUTPUT ${NUM_WORKERS_PER_OUTPUT} |
| 17 | + |
| 18 | +// Maximum total threads in a work group |
| 19 | +#define MAX_THREADS 256 |
| 20 | + |
| 21 | +${define_active_storage_type(STORAGE)} |
| 22 | +${define_required_extensions("int8")} |
| 23 | + |
| 24 | +#extension GL_EXT_control_flow_attributes : require |
| 25 | + |
| 26 | +layout(std430) buffer; |
| 27 | + |
| 28 | +#include "common.glslh" |
| 29 | + |
| 30 | +${layout_declare_tensor(B, "w", "t_scales", "float", "buffer")} |
| 31 | +${layout_declare_tensor(B, "w", "t_zps", "int", "buffer")} |
| 32 | +${layout_declare_tensor(B, "r", "t_input", DTYPE, STORAGE, is_scalar_array=False)} |
| 33 | + |
| 34 | +${layout_declare_ubo(B, "ivec4", "input_sizes")} |
| 35 | + |
| 36 | +layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; |
| 37 | + |
| 38 | +layout(push_constant) uniform PushConstants { |
| 39 | + int quant_min; |
| 40 | + int quant_max; |
| 41 | +}; |
| 42 | + |
| 43 | +// Shared memory for cooperative min/max finding |
| 44 | +shared T shared_min[NUM_OUTPUTS_PER_WG][NUM_WORKERS_PER_OUTPUT]; |
| 45 | +shared T shared_max[NUM_OUTPUTS_PER_WG][NUM_WORKERS_PER_OUTPUT]; |
| 46 | + |
| 47 | +const float SMALL_SCALE_THRESHOLD = 6.1e-5; |
| 48 | + |
| 49 | +void calculate_scale_and_zero_point( |
| 50 | + float min_val, |
| 51 | + float max_val, |
| 52 | + int qmin, |
| 53 | + int qmax, |
| 54 | + out float scale, |
| 55 | + out int8_t zero_point) { |
| 56 | + |
| 57 | + // Extend the [min, max] interval to ensure it contains 0 |
| 58 | + min_val = min(min_val, 0.0); |
| 59 | + max_val = max(max_val, 0.0); |
| 60 | + |
| 61 | + // Calculate scale |
| 62 | + scale = (max_val - min_val) / float(qmax - qmin); |
| 63 | + |
| 64 | + // Handle special cases for scale |
| 65 | + if (scale == 0.0 || isinf(1.0 / scale)) { |
| 66 | + scale = 0.1; |
| 67 | + } |
| 68 | + |
| 69 | + // Cut off small scale |
| 70 | + if (scale < SMALL_SCALE_THRESHOLD) { |
| 71 | + float org_scale = scale; |
| 72 | + scale = SMALL_SCALE_THRESHOLD; |
| 73 | + // Adjust the min and max based on the new scale |
| 74 | + if (min_val == 0.0) { |
| 75 | + max_val = SMALL_SCALE_THRESHOLD * float(qmax - qmin); |
| 76 | + } else if (max_val == 0.0) { |
| 77 | + min_val = -SMALL_SCALE_THRESHOLD * float(qmax - qmin); |
| 78 | + } else { |
| 79 | + float amplifier = SMALL_SCALE_THRESHOLD / org_scale; |
| 80 | + min_val *= amplifier; |
| 81 | + max_val *= amplifier; |
| 82 | + } |
| 83 | + } |
| 84 | + |
| 85 | + // Zero-point computation |
| 86 | + float zero_point_from_min = float(qmin) - min_val / scale; |
| 87 | + float zero_point_from_max = float(qmax) - max_val / scale; |
| 88 | + float zero_point_from_min_error = abs(float(qmin)) - abs(min_val / scale); |
| 89 | + float zero_point_from_max_error = abs(float(qmax)) - abs(max_val / scale); |
| 90 | + |
| 91 | + float initial_zero_point = zero_point_from_min_error < zero_point_from_max_error |
| 92 | + ? zero_point_from_min |
| 93 | + : zero_point_from_max; |
| 94 | + |
| 95 | + // Nudge zero point to be an integer |
| 96 | + int nudged_zero_point; |
| 97 | + if (initial_zero_point < float(qmin)) { |
| 98 | + nudged_zero_point = qmin; |
| 99 | + } else if (initial_zero_point > float(qmax)) { |
| 100 | + nudged_zero_point = qmax; |
| 101 | + } else { |
| 102 | + nudged_zero_point = int(round(initial_zero_point)); |
| 103 | + } |
| 104 | + |
| 105 | + zero_point = int8_t(nudged_zero_point); |
| 106 | +} |
| 107 | + |
| 108 | +#ifdef USING_BUFFER |
| 109 | + |
| 110 | +VEC4_T load_input_x4(const int x4, const int y, const int ntexels_x) { |
| 111 | + return t_input[(y * ntexels_x) + x4]; |
| 112 | +} |
| 113 | + |
| 114 | +#else // USING_TEXTURE |
| 115 | + |
| 116 | +VEC4_T load_input_x4(const int x4, const int y, const int ntexels_x) { |
| 117 | + return texelFetch(t_input, ivec3(x4, y, 0), 0); |
| 118 | +} |
| 119 | + |
| 120 | +#endif // USING_BUFFER |
| 121 | + |
| 122 | +void main() { |
| 123 | + const int worker_id = int(gl_LocalInvocationID.x); |
| 124 | + const int output_id = int(gl_LocalInvocationID.y); |
| 125 | + |
| 126 | + const int output_y = int(gl_GlobalInvocationID.y); |
| 127 | + |
| 128 | + if (output_y >= input_sizes.y) { |
| 129 | + return; |
| 130 | + } |
| 131 | + |
| 132 | + // Input is 2D tensor (height x width), width-packed |
| 133 | + // Each channel corresponds to a row in the tensor |
| 134 | + const int X4 = div_4(input_sizes.x); |
| 135 | + |
| 136 | + // Initialize thread-local min/max |
| 137 | + float local_min = 1e30; |
| 138 | + float local_max = -1e30; |
| 139 | + |
| 140 | + // Each thread processes elements along their assigned output_id with stride |
| 141 | + // NUM_WORKERS_PER_OUTPUT |
| 142 | + for (int x4 = worker_id; x4 < X4; x4 += NUM_WORKERS_PER_OUTPUT) { |
| 143 | + VEC4_T in_texel = load_input_x4(x4, output_y, X4); |
| 144 | + for (int i = 0; i < 4; i++) { |
| 145 | + local_min = min(local_min, in_texel[i]); |
| 146 | + local_max = max(local_max, in_texel[i]); |
| 147 | + } |
| 148 | + } |
| 149 | + |
| 150 | + // Store thread-local results in shared memory |
| 151 | + shared_min[output_id][worker_id] = local_min; |
| 152 | + shared_max[output_id][worker_id] = local_max; |
| 153 | + |
| 154 | + memoryBarrierShared(); |
| 155 | + barrier(); |
| 156 | + |
| 157 | + // Tree reduction to compute the overall result |
| 158 | + for (int i = NUM_WORKERS_PER_OUTPUT / 2; i > 0; i >>= 1) { |
| 159 | + if (worker_id < i) { |
| 160 | + shared_min[output_id][worker_id] = min( |
| 161 | + shared_min[output_id][worker_id], |
| 162 | + shared_min[output_id][worker_id + i]); |
| 163 | + shared_max[output_id][worker_id] = max( |
| 164 | + shared_max[output_id][worker_id], |
| 165 | + shared_max[output_id][worker_id + i]); |
| 166 | + } |
| 167 | + memoryBarrierShared(); |
| 168 | + barrier(); |
| 169 | + } |
| 170 | + |
| 171 | + // Only first thread will write out result |
| 172 | + if (worker_id == 0) { |
| 173 | + local_min = shared_min[output_id][0]; |
| 174 | + local_max = shared_max[output_id][0]; |
| 175 | + |
| 176 | + float scale; |
| 177 | + int8_t zero_point; |
| 178 | + calculate_scale_and_zero_point( |
| 179 | + local_min, local_max, quant_min, quant_max, scale, zero_point); |
| 180 | + |
| 181 | + t_scales[output_y] = scale; |
| 182 | + t_zps[output_y] = zero_point; |
| 183 | + } |
| 184 | +} |
0 commit comments