Support for RT_GLOBAL_BUFFER_POINTER implicit arg in GlobalStateBuffer

There are several cases when calling `intel_get_implicit_dispatch_globals`
builtin from function `F` causes a crash:
1. Function F definition is compiled in ModuleA, but called through stackcall in ModuleB.
2. Function F has "referenced-indirectly" attribute.
3. Function F is a recurrent function.

In these three cases, `RT_GLOBAL_BUFFER_POINTER` implicit argument must
be passed through GlobalStateBuffer, since 'F' is called through a stackcall.

Author: aratajew

IGC/AdaptorCommon/ImplicitArgs.cpp

@@ -90,7 +90,7 @@ static const std::vector<ImplicitArg> IMPLICIT_ARGS = {
     ImplicitArg(ImplicitArg::SYNC_BUFFER, "syncBuffer", ImplicitArg::GLOBALPTR, WIAnalysis::UNIFORM_GLOBAL, 1, ImplicitArg::ALIGN_PTR, false, GenISAIntrinsic::GenISA_getSyncBuffer),
 
     // raytracing
-    ImplicitArg(ImplicitArg::RT_GLOBAL_BUFFER_POINTER, "globalPointer", ImplicitArg::GLOBALPTR, WIAnalysis::UNIFORM_GLOBAL, 1, ImplicitArg::ALIGN_GRF, false),
+    ImplicitArg(ImplicitArg::RT_GLOBAL_BUFFER_POINTER, "globalPointer", ImplicitArg::GLOBALPTR, WIAnalysis::UNIFORM_GLOBAL, 1, ImplicitArg::ALIGN_GRF, false, GenISAIntrinsic::GenISA_getRtGlobalBufferPtr),
     ImplicitArg(ImplicitArg::RT_LOCAL_BUFFER_POINTER, "localPointer", ImplicitArg::GLOBALPTR, WIAnalysis::UNIFORM_GLOBAL, 1, ImplicitArg::ALIGN_QWORD, false),
     ImplicitArg(ImplicitArg::RT_INLINED_DATA, "inlinedData", ImplicitArg::GLOBALPTR, WIAnalysis::UNIFORM_GLOBAL, 2, ImplicitArg::ALIGN_GRF, false),
     ImplicitArg(ImplicitArg::RT_STACK_ID, "stackID", ImplicitArg::SHORT, WIAnalysis::RANDOM, 16, ImplicitArg::ALIGN_GRF, false),

IGC/AdaptorCommon/RayTracing/RayTracingIntrinsicResolution.cpp

@@ -49,11 +49,12 @@ class RayTracingIntrinsicResolution : public FunctionPass, public InstVisitor<Ra
 
     static char ID;
 private:
-    Argument* getImplicitArg(Function *F, ImplicitArg::ArgType argType);
+    Value* getImplicitArg(Function *F, ImplicitArg::ArgType argType);
 private:
     ImplicitArgs m_implicitArgs;
     bool Changed;
     CodeGenContext* m_CGCtx = nullptr;
+    IGCMD::MetaDataUtils* m_pMdUtils = nullptr;
 };
 
 #define PASS_FLAG "raytracing-intrinsic-resolution"
@@ -64,10 +65,10 @@ IGC_INITIALIZE_PASS_BEGIN(RayTracingIntrinsicResolution, PASS_FLAG, PASS_DESCRIP
 IGC_INITIALIZE_PASS_DEPENDENCY(MetaDataUtilsWrapper)
 IGC_INITIALIZE_PASS_END(RayTracingIntrinsicResolution, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)
 
-Argument* RayTracingIntrinsicResolution::getImplicitArg(
+Value* RayTracingIntrinsicResolution::getImplicitArg(
     Function *F, ImplicitArg::ArgType argType)
 {
-    return m_implicitArgs.getImplicitArg(*F, argType);
+    return m_implicitArgs.getImplicitArgValue(*F, argType, m_pMdUtils);
 }
 
 RayTracingIntrinsicResolution::RayTracingIntrinsicResolution() : FunctionPass(ID) {
@@ -121,16 +122,16 @@ void RayTracingIntrinsicResolution::visitCallInst(CallInst &CI)
 
 bool RayTracingIntrinsicResolution::runOnFunction(Function &F)
 {
-    auto *pMdUtils = getAnalysis<MetaDataUtilsWrapper>().getMetaDataUtils();
+    m_pMdUtils = getAnalysis<MetaDataUtilsWrapper>().getMetaDataUtils();
     m_CGCtx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
 
-    if (pMdUtils->findFunctionsInfoItem(&F) == pMdUtils->end_FunctionsInfo())
+    if (m_pMdUtils->findFunctionsInfoItem(&F) == m_pMdUtils->end_FunctionsInfo())
         return false;
 
     Changed = false;
     m_implicitArgs = ImplicitArgs(
         F,
-        pMdUtils);
+        m_pMdUtils);
     visit(F);
 
     return Changed;

IGC/AdaptorOCL/OCL/sp/zebin_builder.cpp

@@ -564,6 +564,7 @@ void ZEBinaryBuilder::addKernelExecEnv(const SOpenCLKernelInfo& annotations,
     }
     env.eu_thread_count = annotations.m_executionEnvironment.numThreads;
     env.has_sample = annotations.m_executionEnvironment.HasSample;
+    env.has_rtcalls = annotations.m_executionEnvironment.HasRTCalls;
 }
 
 void ZEBinaryBuilder::addFunctionExecEnv(const SOpenCLKernelInfo& annotations,
@@ -576,6 +577,7 @@ void ZEBinaryBuilder::addFunctionExecEnv(const SOpenCLKernelInfo& annotations,
     env.grf_count = annotations.m_executionEnvironment.NumGRFRequired;
     env.simd_size = annotations.m_executionEnvironment.CompiledSIMDSize;
     env.barrier_count = zeFuncAttr.f_BarrierCount;
+    env.has_rtcalls = zeFuncAttr.f_hasRTCalls;
 }
 
 void ZEBinaryBuilder::addLocalIds(uint32_t simdSize, uint32_t grfSize,

IGC/Compiler/CISACodeGen/CISABuilder.cpp

@@ -5686,6 +5686,7 @@ namespace IGC
     }
 
     void CEncoder::CreateFuncAttributeTable(VISAKernel* pMainKernel) {
+        ModuleMetaData* modMD = m_program->GetContext()->getModuleMetaData();
         SProgramOutput *pOutput = m_program->ProgramOutput();
         void *&buffer = pOutput->m_funcAttributeTable;
         unsigned &bufferSize = pOutput->m_funcAttributeTableSize;
@@ -5727,8 +5728,9 @@ namespace IGC
             uint8_t isExternal = F->hasFnAttribute("referenced-indirectly") ? 1 : 0;
             // Set per-function barrier count from vISA information.
             uint32_t barrierCnt = jitInfo->numBarriers;
+            uint8_t hasRTCalls = (uint8_t)modMD->FuncMD[F].hasSyncRTCalls;
             attrs.emplace_back(entry.f_isKernel, isExternal, barrierCnt, entry.f_privateMemPerThread,
-                entry.f_spillMemPerThread, F->getName().str());
+                entry.f_spillMemPerThread, F->getName().str(), hasRTCalls);
             attribTable.push_back(entry);
         }
 

IGC/Compiler/CISACodeGen/EmitVISAPass.cpp

@@ -8030,6 +8030,7 @@ void EmitPass::EmitGenIntrinsicMessage(llvm::GenIntrinsicInst* inst)
     case GenISAIntrinsic::GenISA_getStageInGridOrigin:
     case GenISAIntrinsic::GenISA_getStageInGridSize:
     case GenISAIntrinsic::GenISA_getSyncBuffer:
+    case GenISAIntrinsic::GenISA_getRtGlobalBufferPtr:
         emitImplicitArgIntrinsic(inst);
         break;
     case GenISAIntrinsic::GenISA_AsyncStackID:

IGC/Compiler/CISACodeGen/WIAnalysis.cpp

@@ -1352,6 +1352,7 @@ WIAnalysis::WIDependancy WIAnalysisRunner::calculate_dep(const CallInst* inst)
         GII_id == GenISAIntrinsic::GenISA_getStageInGridOrigin ||
         GII_id == GenISAIntrinsic::GenISA_getStageInGridSize ||
         GII_id == GenISAIntrinsic::GenISA_getSyncBuffer ||
+        GII_id == GenISAIntrinsic::GenISA_getRtGlobalBufferPtr ||
         GII_id == GenISAIntrinsic::GenISA_GetImplicitBufferPtr ||
         GII_id == GenISAIntrinsic::GenISA_GetLocalIdBufferPtr  ||
         GII_id == GenISAIntrinsic::GenISA_staticConstantPatchValue)
@@ -1393,6 +1394,7 @@ WIAnalysis::WIDependancy WIAnalysisRunner::calculate_dep(const CallInst* inst)
         case GenISAIntrinsic::GenISA_getStageInGridOrigin:
         case GenISAIntrinsic::GenISA_getStageInGridSize:
         case GenISAIntrinsic::GenISA_getSyncBuffer:
+        case GenISAIntrinsic::GenISA_getRtGlobalBufferPtr:
             return ImplicitArgs::getArgDep(GII_id);
         }
 

IGC/Compiler/Optimizer/OpenCLPasses/WIFuncs/WIFuncResolution.cpp

@@ -18,6 +18,8 @@ SPDX-License-Identifier: MIT
 #include <llvmWrapper/Support/Alignment.h>
 #include <llvmWrapper/IR/DerivedTypes.h>
 
+#include <cstddef>
+
 using namespace llvm;
 using namespace IGC;
 
@@ -213,6 +215,22 @@ void WIFuncResolution::visitCallInst(CallInst& CI)
 
 *************************************************************************************************/
 
+#pragma pack(push, 1)
+namespace packed
+{
+#include "implicit_args_struct.h"
+}
+#pragma pack(pop)
+
+#undef IMPLICIT_ARGS_STRUCT_H_
+#include "implicit_args_struct.h"
+
+// According to the ABI specification, implicit_args struct must be naturally aligned.
+// To ensure that offsets to struct members are compiler-independent, it is necessary to
+// align struct member offset to be divisible by the size of the member. It implicates
+// inserting an additional padding member in some cases.
+static_assert(sizeof(packed::implicit_args) == sizeof(implicit_args), "Implicit args struct is not properly aligned!");
+
 // Structure of side buffer generated by NEO:
 //struct implicit_args {
 //    uint8_t struct_size;
@@ -233,6 +251,8 @@ void WIFuncResolution::visitCallInst(CallInst& CI)
 //    uint32_t group_count_x;
 //    uint32_t group_count_y;
 //    uint32_t group_count_z;
+//    uint32_t padding0;
+//    uint64_t rt_global_buffer_ptr;
 //};
 
 // For SIMD8:
@@ -259,46 +279,6 @@ void WIFuncResolution::visitCallInst(CallInst& CI)
 //    uint16_t lz[32];
 //};
 
-
-class GLOBAL_STATE_FIELD_OFFSETS
-{
-public:
-    // This class holds offsets of various fields in side buffer
-    static const uint32_t STRUCT_SIZE = 0;
-
-    static const uint32_t VERSION = STRUCT_SIZE + sizeof(uint8_t);
-
-    static const uint32_t NUM_WORK_DIM = VERSION + sizeof(uint8_t);
-
-    static const uint32_t SIMDSIZE = NUM_WORK_DIM + sizeof(uint8_t);
-
-    static const uint32_t LOCAL_SIZES = SIMDSIZE + sizeof(uint8_t);
-    static const uint32_t LOCAL_SIZE_X = LOCAL_SIZES;
-    static const uint32_t LOCAL_SIZE_Y = LOCAL_SIZE_X + sizeof(uint32_t);
-    static const uint32_t LOCAL_SIZE_Z = LOCAL_SIZE_Y + sizeof(uint32_t);
-
-    static const uint32_t GLOBAL_SIZES = LOCAL_SIZE_Z + sizeof(uint32_t);
-    static const uint32_t GLOBAL_SIZE_X = GLOBAL_SIZES;
-    static const uint32_t GLOBAL_SIZE_Y = GLOBAL_SIZE_X + sizeof(uint64_t);
-    static const uint32_t GLOBAL_SIZE_Z = GLOBAL_SIZE_Y + sizeof(uint64_t);
-
-    static const uint32_t PRINTF_BUFFER = GLOBAL_SIZE_Z + sizeof(uint64_t);
-
-    static const uint32_t GLOBAL_OFFSETS = PRINTF_BUFFER + sizeof(uint64_t);
-    static const uint32_t GLOBAL_OFFSET_X = GLOBAL_OFFSETS;
-    static const uint32_t GLOBAL_OFFSET_Y = GLOBAL_OFFSET_X + sizeof(uint64_t);
-    static const uint32_t GLOBAL_OFFSET_Z = GLOBAL_OFFSET_Y + sizeof(uint64_t);
-
-    static const uint32_t LOCAL_IDS = GLOBAL_OFFSET_Z + sizeof(uint64_t);
-
-    static const uint32_t GROUP_COUNTS = LOCAL_IDS + sizeof(uint64_t);
-    static const uint32_t GROUP_COUNT_X = GROUP_COUNTS;
-    static const uint32_t GROUP_COUNT_Y = GROUP_COUNT_X + sizeof(uint32_t);
-    static const uint32_t GROUP_COUNT_Z = GROUP_COUNT_Y + sizeof(uint32_t);
-
-    static const uint32_t TOTAL_SIZE = GROUP_COUNT_Z + sizeof(uint32_t);
-};
-
 llvm::Value* LowerImplicitArgIntrinsics::BuildLoadInst(llvm::CallInst& CI, unsigned int Offset, llvm::Type* DataType)
 {
     // This function computes type aligned address that includes Offset.
@@ -357,9 +337,9 @@ llvm::Value* LowerImplicitArgIntrinsics::BuildLoadInst(llvm::CallInst& CI, unsig
 
     if (Offset != AlignedOffset)
     {
-        auto ByteType = Type::getInt8Ty(Builder.getContext());
-        auto BitCastToByte = Builder.CreateBitCast(LoadedData, ByteType);
-        Value* NewVector = UndefValue::get(IGCLLVM::FixedVectorType::get(ByteType, Size));
+        auto ByteVectorType = IGCLLVM::FixedVectorType::get(Builder.getInt8Ty(), LoadByteSize);
+        auto BitCastToByte = Builder.CreateBitCast(LoadedData, ByteVectorType);
+        Value* NewVector = UndefValue::get(IGCLLVM::FixedVectorType::get(Builder.getInt8Ty(), Size));
         for (unsigned int I = Offset; I != (Offset + Size); ++I)
         {
             auto Elem = Builder.CreateExtractElement(BitCastToByte, I - AlignedOffset);
@@ -790,7 +770,7 @@ void LowerImplicitArgIntrinsics::visitCallInst(CallInst& CI)
 
                 // Get Local ID Base Ptr
                 auto DataTypeI64 = Type::getInt64Ty(F->getParent()->getContext());
-                unsigned int Offset = GLOBAL_STATE_FIELD_OFFSETS::LOCAL_IDS;
+                unsigned int Offset = offsetof(implicit_args, local_id_table_ptr);
                 auto LocalIDBase = BuildLoadInst(CI, Offset, DataTypeI64);
 
                 // Get local thread id
@@ -842,7 +822,7 @@ void LowerImplicitArgIntrinsics::visitCallInst(CallInst& CI)
             // Assume local size and enqueued local size are the same
             auto ElemTypeD = Type::getInt32Ty(F->getParent()->getContext());
             auto VecTyD = IGCLLVM::FixedVectorType::get(ElemTypeD, 3);
-            unsigned int Offset = GLOBAL_STATE_FIELD_OFFSETS::LOCAL_SIZE_X;
+            unsigned int Offset = offsetof(implicit_args, local_size_x);
             auto LoadInst = BuildLoadInst(CI, Offset, VecTyD);
             V = LoadInst;
             break;
@@ -851,7 +831,7 @@ void LowerImplicitArgIntrinsics::visitCallInst(CallInst& CI)
         {
             // global_offset is loaded from PayloadHeader[0:2]
             // currently there are no other uses for payload header.
-            unsigned int Offset = GLOBAL_STATE_FIELD_OFFSETS::GLOBAL_OFFSET_X;
+            unsigned int Offset = offsetof(implicit_args, global_offset_x);
             auto ElemTypeD = Type::getInt32Ty(F->getParent()->getContext());
             auto VecTyQ = IGCLLVM::FixedVectorType::get(Type::getInt64Ty(F->getParent()->getContext()), 3);
             auto LoadInst = BuildLoadInst(CI, Offset, VecTyQ);
@@ -868,7 +848,7 @@ void LowerImplicitArgIntrinsics::visitCallInst(CallInst& CI)
         case GenISAIntrinsic::GenISA_getGlobalSize:
         case GenISAIntrinsic::GenISA_getStageInGridSize:
         {
-            unsigned int Offset = GLOBAL_STATE_FIELD_OFFSETS::GLOBAL_SIZE_X;
+            unsigned int Offset = offsetof(implicit_args, global_size_x);
             auto VecTyQ = IGCLLVM::FixedVectorType::get(Type::getInt64Ty(F->getParent()->getContext()), 3);
             auto ElemTypeD = Type::getInt32Ty(F->getParent()->getContext());
             auto LoadInst = BuildLoadInst(CI, Offset, VecTyQ);
@@ -886,15 +866,15 @@ void LowerImplicitArgIntrinsics::visitCallInst(CallInst& CI)
         {
             auto ElemTypeUD = Type::getInt32Ty(F->getParent()->getContext());
             auto VecTyUD = IGCLLVM::FixedVectorType::get(ElemTypeUD, 3);
-            unsigned int Offset = GLOBAL_STATE_FIELD_OFFSETS::GROUP_COUNT_X;
+            unsigned int Offset = offsetof(implicit_args, group_count_x);
             auto LoadInst = BuildLoadInst(CI, Offset, VecTyUD);
             V = LoadInst;
             break;
         }
         case GenISAIntrinsic::GenISA_getWorkDim:
         {
             unsigned int Size = 4;
-            unsigned int Offset = GLOBAL_STATE_FIELD_OFFSETS::NUM_WORK_DIM / Size;
+            unsigned int Offset = offsetof(implicit_args, num_work_dim) / Size;
             auto TypeUD = Type::getInt32Ty(F->getParent()->getContext());
             auto LoadInst = BuildLoadInst(CI, Offset, TypeUD);
             auto LShr = Builder.CreateLShr(LoadInst, (uint64_t)16);
@@ -906,12 +886,20 @@ void LowerImplicitArgIntrinsics::visitCallInst(CallInst& CI)
         {
             // This function is invoked when expanding printf call to retrieve printf buffer ptr.
             auto DataTypeI64 = Type::getInt64Ty(CI.getFunction()->getParent()->getContext());
-            unsigned int Offset = GLOBAL_STATE_FIELD_OFFSETS::PRINTF_BUFFER;
+            unsigned int Offset = offsetof(implicit_args, printf_buffer_ptr);
             auto Result = BuildLoadInst(CI, Offset, DataTypeI64);
             Result = Builder.CreateIntToPtr(Result, CI.getType());
             V = Result;
             break;
         }
+        case GenISAIntrinsic::GenISA_getRtGlobalBufferPtr:
+        {
+            unsigned int Offset = offsetof(implicit_args, rt_global_buffer_ptr);
+            auto Result = BuildLoadInst(CI, Offset, Builder.getInt64Ty());
+            Result = Builder.CreateIntToPtr(Result, CI.getType());
+            V = Result;
+            break;
+        }
         default:
             break;
         }

IGC/Compiler/Optimizer/OpenCLPasses/WIFuncs/implicit_args_struct.h

@@ -0,0 +1,27 @@
+#ifndef IMPLICIT_ARGS_STRUCT_H_
+#define IMPLICIT_ARGS_STRUCT_H_
+
+struct implicit_args {
+    uint8_t struct_size;
+    uint8_t struct_version;
+    uint8_t num_work_dim;
+    uint8_t simd_width;
+    uint32_t local_size_x;
+    uint32_t local_size_y;
+    uint32_t local_size_z;
+    uint64_t global_size_x;
+    uint64_t global_size_y;
+    uint64_t global_size_z;
+    uint64_t printf_buffer_ptr;
+    uint64_t global_offset_x;
+    uint64_t global_offset_y;
+    uint64_t global_offset_z;
+    uint64_t local_id_table_ptr;
+    uint32_t group_count_x;
+    uint32_t group_count_y;
+    uint32_t group_count_z;
+    uint32_t padding0;
+    uint64_t rt_global_buffer_ptr;
+};
+
+#endif // IMPLICIT_ARGS_STRUCT_H_

IGC/GenISAIntrinsics/Intrinsic_definitions.py

@@ -1231,6 +1231,11 @@
     [],
     "None"]],
 ####################################################################################################
+"GenISA_getRtGlobalBufferPtr": ["",
+    [("anyptr",                           "result"),
+    [],
+    "None"]],
+####################################################################################################
 "GenISA_getSR0": ["sr0.# the state register",
     [("int",                           "result"),
     [("int",                           "index/offset of the subregister within sr0")],

IGC/ZEBinWriter/zebin/source/autogen/ZEInfo.hpp

@@ -1,6 +1,6 @@
 /*========================== begin_copyright_notice ============================
 
-Copyright (C) 2020-2022 Intel Corporation
+Copyright (C) 2020-2023 Intel Corporation
 
 SPDX-License-Identifier: MIT
 
@@ -45,7 +45,7 @@ struct zeInfoExecutionEnv
 {
     bool operator==(const zeInfoExecutionEnv& other) const
     {
-        return barrier_count == other.barrier_count && disable_mid_thread_preemption == other.disable_mid_thread_preemption && grf_count == other.grf_count && has_4gb_buffers == other.has_4gb_buffers && has_device_enqueue == other.has_device_enqueue && has_dpas == other.has_dpas && has_fence_for_image_access == other.has_fence_for_image_access && has_global_atomics == other.has_global_atomics && has_multi_scratch_spaces == other.has_multi_scratch_spaces && has_no_stateless_write == other.has_no_stateless_write && has_stack_calls == other.has_stack_calls && require_disable_eufusion == other.require_disable_eufusion && indirect_stateless_count == other.indirect_stateless_count && inline_data_payload_size == other.inline_data_payload_size && offset_to_skip_per_thread_data_load == other.offset_to_skip_per_thread_data_load && offset_to_skip_set_ffid_gp == other.offset_to_skip_set_ffid_gp && required_sub_group_size == other.required_sub_group_size && required_work_group_size == other.required_work_group_size && simd_size == other.simd_size && slm_size == other.slm_size && subgroup_independent_forward_progress == other.subgroup_independent_forward_progress && thread_scheduling_mode == other.thread_scheduling_mode && work_group_walk_order_dimensions == other.work_group_walk_order_dimensions && eu_thread_count == other.eu_thread_count && has_sample == other.has_sample;
+        return barrier_count == other.barrier_count && disable_mid_thread_preemption == other.disable_mid_thread_preemption && grf_count == other.grf_count && has_4gb_buffers == other.has_4gb_buffers && has_device_enqueue == other.has_device_enqueue && has_dpas == other.has_dpas && has_fence_for_image_access == other.has_fence_for_image_access && has_global_atomics == other.has_global_atomics && has_multi_scratch_spaces == other.has_multi_scratch_spaces && has_no_stateless_write == other.has_no_stateless_write && has_stack_calls == other.has_stack_calls && require_disable_eufusion == other.require_disable_eufusion && indirect_stateless_count == other.indirect_stateless_count && inline_data_payload_size == other.inline_data_payload_size && offset_to_skip_per_thread_data_load == other.offset_to_skip_per_thread_data_load && offset_to_skip_set_ffid_gp == other.offset_to_skip_set_ffid_gp && required_sub_group_size == other.required_sub_group_size && required_work_group_size == other.required_work_group_size && simd_size == other.simd_size && slm_size == other.slm_size && subgroup_independent_forward_progress == other.subgroup_independent_forward_progress && thread_scheduling_mode == other.thread_scheduling_mode && work_group_walk_order_dimensions == other.work_group_walk_order_dimensions && eu_thread_count == other.eu_thread_count && has_sample == other.has_sample && has_rtcalls == other.has_rtcalls;
     }
     zeinfo_int32_t barrier_count = 0;
     zeinfo_bool_t disable_mid_thread_preemption = false;
@@ -72,6 +72,7 @@ struct zeInfoExecutionEnv
     std::vector<zeinfo_int32_t> work_group_walk_order_dimensions;
     zeinfo_int32_t eu_thread_count = 0;
     zeinfo_bool_t has_sample = false;
+    zeinfo_bool_t has_rtcalls = false;
 };
 struct zeInfoPayloadArgument
 {