File size: 3,951 Bytes
57c3a10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 | /*
* Copyright (c) 2025 by FlashInfer team.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FLASHINFER_ATTENTION_PERSISTENT_TEMPLATE_CUH
#define FLASHINFER_ATTENTION_PERSISTENT_TEMPLATE_CUH
#include <cooperative_groups.h>
#include <cuda_runtime.h>
#include <cstdint>
#include "../profiler.cuh"
namespace flashinfer {
namespace cg = cooperative_groups;
// Define profiler event types for persistent kernels
enum class PersistentProfileEventType {
kRunner1 = 0U,
kRunner2 = 1U,
kReduction = 2U,
};
struct ProfilerClosure {
PROFILER_CLOSURE_PARAMS_DECL
};
// Helper metafunction to find maximum threads among multiple BlockPersistentRunners
template <typename... Runners>
struct max_threads;
template <typename Runner>
struct max_threads<Runner> {
static constexpr size_t value = Runner::KTraits::NUM_THREADS;
};
template <typename Runner1, typename Runner2, typename... RestRunners>
struct max_threads<Runner1, Runner2, RestRunners...> {
static constexpr size_t value = Runner1::KTraits::NUM_THREADS > Runner2::KTraits::NUM_THREADS
? max_threads<Runner1, RestRunners...>::value
: max_threads<Runner2, RestRunners...>::value;
};
// Two runners version
template <class BlockPersistentRunner1, class BlockPersistentRunner2, class BlockReductionRunner>
__global__ __launch_bounds__(
max_threads<BlockPersistentRunner1, BlockPersistentRunner2, BlockReductionRunner>::
value) void PersistentKernelTemplate(const __grid_constant__
typename BlockPersistentRunner1::Params params_1,
const __grid_constant__
typename BlockPersistentRunner2::Params params_2) {
extern __shared__ uint8_t smem[];
#ifdef FLASHINFER_ENABLE_PROFILER
ProfilerClosure
profiler_closure; // no volatile as this is scope.CTA, and only threadIdx == 0 is modifying
PROFILER_INIT(params_1, smem, profiler_closure, 0, 1, (threadIdx.x == 0));
#endif
auto& smem_storage_1 =
reinterpret_cast<typename BlockPersistentRunner1::KTraits::SharedStorage&>(smem);
auto& smem_storage_2 =
reinterpret_cast<typename BlockPersistentRunner2::KTraits::SharedStorage&>(smem);
auto grid = cg::this_grid();
#ifndef FLASHINFER_ENABLE_PROFILER
BlockPersistentRunner1::Run(params_1, &smem_storage_1);
BlockPersistentRunner2::Run(params_2, &smem_storage_2);
grid.sync();
BlockReductionRunner::Run(params_1.partial_o, params_1.final_o, params_1.partial_lse,
params_1.final_lse, *(params_1.num_packed_qo_len),
params_1.gqa_group_size, params_1.num_kv_heads, params_1.merge_indptr,
params_1.merge_o_indices, smem);
#else
BlockPersistentRunner1::Run(params_1, &smem_storage_1, profiler_closure);
BlockPersistentRunner2::Run(params_2, &smem_storage_2, profiler_closure);
grid.sync();
BlockReductionRunner::Run(params_1.partial_o, params_1.final_o, params_1.partial_lse,
params_1.final_lse, *(params_1.num_packed_qo_len),
params_1.gqa_group_size, params_1.num_kv_heads, params_1.merge_indptr,
params_1.merge_o_indices, smem, profiler_closure);
#endif
}
} // namespace flashinfer
#endif // FLASHINFER_ATTENTION_PERSISTENT_TEMPLATE_CUH
|