Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions level_zero/core/source/context/context_imp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1838,27 +1838,12 @@ bool ContextImp::isAllocationSuitableForCompression(const StructuresLookupTable
auto &hwInfo = device.getHwInfo();
auto &gfxCoreHelper = device.getGfxCoreHelper();
auto neoDevice = device.getNEODevice();
auto rootDeviceIndex = neoDevice->getRootDeviceIndex();
auto &l0GfxCoreHelper = neoDevice->getRootDeviceEnvironment().getHelper<L0GfxCoreHelper>();
auto peerAccess = neoDevice->hasAnyPeerAccess();

if (!l0GfxCoreHelper.usmCompressionSupported(hwInfo) || !gfxCoreHelper.isBufferSizeSuitableForCompression(allocSize) || structuresLookupTable.uncompressedHint) {
return false;
}

if (peerAccess.value_or(false)) {
for (auto &dev : this->driverHandle->devices) {
if (rootDeviceIndex == dev->getRootDeviceIndex() || !l0GfxCoreHelper.usmCompressionSupported(dev->getHwInfo())) {
continue;
}
ze_bool_t canAccess = false;
ze_result_t checkResult = device.canAccessPeer(dev, &canAccess);
if (checkResult == ZE_RESULT_SUCCESS && canAccess) {
return false;
}
}
}

if (l0GfxCoreHelper.forceDefaultUsmCompressionSupport()) {
return true;
}
Expand Down
2 changes: 2 additions & 0 deletions level_zero/core/source/driver/driver_handle.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -909,6 +909,8 @@ NEO::GraphicsAllocation *DriverHandle::getPeerAllocation(Device *device,
if (numHandles > 1) {
peerAllocRootDeviceIndex = device->getNEODevice()->getRootDevice()->getRootDeviceIndex();
}
auto &l0GfxCoreHelper = device->getNEODevice()->getRootDeviceEnvironment().getHelper<L0GfxCoreHelper>();
l0GfxCoreHelper.p2pDecompressBufferIfRequired(alloc, device->getDriverHandle());
NEO::SvmAllocationData allocDataInternal(peerAllocRootDeviceIndex);

if (numHandles > 1) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2025 Intel Corporation
* Copyright (C) 2020-2026 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
Expand Down Expand Up @@ -135,6 +135,9 @@ ze_record_replay_graph_exp_flags_t L0GfxCoreHelper::getRecordReplayGraphCapabili
return l0GfxCoreHelper.getPlatformRecordReplayGraphCapabilities();
}

void L0GfxCoreHelper::p2pDecompressBufferIfRequired(NEO::GraphicsAllocation *alloc, DriverHandle *driverHandle) const {
}

} // namespace L0

template <>
Expand Down
4 changes: 4 additions & 0 deletions level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "shared/source/helpers/device_bitfield.h"
#include "shared/source/helpers/heap_base_address_model.h"

#include "level_zero/core/source/driver/driver_handle.h"
#include "level_zero/driver_experimental/zex_graph.h"
#include "level_zero/tools/source/debug/eu_thread.h"
#include "level_zero/zet_intel_gpu_debug.h"
Expand Down Expand Up @@ -46,6 +47,7 @@ enum class RTASDeviceFormatInternal {
struct CopyOffloadMode;
struct Event;
struct Device;
struct DriverHandleImp;
struct EventPool;
struct EventDescriptor;

Expand Down Expand Up @@ -123,6 +125,7 @@ class L0GfxCoreHelper : public NEO::ApiGfxCoreHelper {
virtual size_t getMaxFillPatternSizeForCopyEngine() const = 0;
virtual uint64_t getSupportedCustomOperations1() const = 0;
virtual uint64_t getSupportedCustomOperations2() const = 0;
virtual void p2pDecompressBufferIfRequired(NEO::GraphicsAllocation *alloc, L0::DriverHandle *driverHandle) const = 0;

protected:
L0GfxCoreHelper() = default;
Expand Down Expand Up @@ -186,6 +189,7 @@ class L0GfxCoreHelperHw : public L0GfxCoreHelper {
size_t getMaxFillPatternSizeForCopyEngine() const override;
uint64_t getSupportedCustomOperations1() const override;
uint64_t getSupportedCustomOperations2() const override;
void p2pDecompressBufferIfRequired(NEO::GraphicsAllocation *alloc, L0::DriverHandle *driverHandle) const override;

protected:
L0GfxCoreHelperHw() = default;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2023-2025 Intel Corporation
* Copyright (C) 2023-2026 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
Expand Down Expand Up @@ -49,4 +49,8 @@ ze_record_replay_graph_exp_flags_t L0GfxCoreHelperHw<Family>::getPlatformRecordR
return 0u;
}

template <typename Family>
void L0GfxCoreHelperHw<Family>::p2pDecompressBufferIfRequired(NEO::GraphicsAllocation *alloc, DriverHandle *driverHandle) const {
}

} // namespace L0
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2025 Intel Corporation
* Copyright (C) 2025-2026 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
Expand Down Expand Up @@ -89,4 +89,9 @@ template <typename Family>
zet_debug_regset_type_intel_gpu_t L0GfxCoreHelperHw<Family>::getRegsetTypeForLargeGrfDetection() const {
return ZET_DEBUG_REGSET_TYPE_SR_INTEL_GPU;
}

template <typename Family>
void L0GfxCoreHelperHw<Family>::p2pDecompressBufferIfRequired(NEO::GraphicsAllocation *alloc, DriverHandle *driverHandle) const {
}

} // namespace L0
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@
*
*/

#include "shared/source/command_stream/command_stream_receiver.h"
#include "shared/source/gmm_helper/gmm.h"
#include "shared/source/helpers/blit_properties.h"
#include "shared/source/memory_manager/memory_operations_handler.h"
#include "shared/source/xe2_hpg_core/hw_cmds.h"

#include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper_base.inl"
Expand Down Expand Up @@ -41,6 +45,25 @@ bool L0GfxCoreHelperHw<Family>::alwaysAllocateEventInLocalMem() const {
return true;
}

template <>
void L0GfxCoreHelperHw<Family>::p2pDecompressBufferIfRequired(NEO::GraphicsAllocation *alloc, DriverHandle *driverHandle) const {
if (not alloc->isCompressionEnabled()) {
return;
}

auto rootDeviceIndex = alloc->getRootDeviceIndex();
auto neoDevice = driverHandle->devices[rootDeviceIndex]->getNEODevice();
auto &rootDeviceEnvironment = neoDevice->getRootDeviceEnvironment();

NEO::MemoryOperationsHandler *memoryOperationsIface = rootDeviceEnvironment.memoryOperationsInterface.get();
if (memoryOperationsIface) {
auto status = memoryOperationsIface->decompress(neoDevice, *alloc);
if (status == NEO::MemoryOperationsStatus::success) {
alloc->getDefaultGmm()->applyAuxFlagsForBuffer(false);
}
}
}

#include "level_zero/core/source/helpers/l0_gfx_core_helper_factory_init.inl"

template class L0GfxCoreHelperHw<Family>;
Expand Down
5 changes: 4 additions & 1 deletion shared/source/execution_environment/execution_environment.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2025 Intel Corporation
* Copyright (C) 2018-2026 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
Expand Down Expand Up @@ -67,6 +67,8 @@ class ExecutionEnvironment : public ReferenceTrackedObject<ExecutionEnvironment>
devicePermissionError = value;
}
bool isDevicePermissionError() const { return devicePermissionError; }
bool isResourceDecompressionEnabled() const { return resourceDecompressionEnabled; };
void setResourceDecompressionEnabled(bool value) { resourceDecompressionEnabled = value; };

void setOneApiPvcWaEnv(bool val) {
oneApiPvcWaEnv = val;
Expand Down Expand Up @@ -104,6 +106,7 @@ class ExecutionEnvironment : public ReferenceTrackedObject<ExecutionEnvironment>
bool fp64EmulationEnabled = false;
bool oneApiPvcWaEnv = true;
bool devicePermissionError = false;
bool resourceDecompressionEnabled = false;

DeviceHierarchyMode deviceHierarchyMode = DeviceHierarchyMode::composite;
DebuggingMode debuggingEnabledMode = DebuggingMode::disabled;
Expand Down
5 changes: 5 additions & 0 deletions shared/source/gmm_helper/gmm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,11 @@ void Gmm::applyAuxFlagsForBuffer(bool preferCompression) {
resourceParams->Flags.Gpu.CCS = 1;
resourceParams->Flags.Gpu.UnifiedAuxSurface = 1;
compressionEnabled = true;
} else {
gfxCoreHelper.applyRenderCompressionFlag(*this, 0);
resourceParams->Flags.Gpu.CCS = 0;
resourceParams->Flags.Gpu.UnifiedAuxSurface = 0;
compressionEnabled = false;
}

PRINT_STRING(debugManager.flags.PrintGmmCompressionParams.get(), stdout,
Expand Down
3 changes: 2 additions & 1 deletion shared/source/memory_manager/memory_operations_handler.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2025 Intel Corporation
* Copyright (C) 2019-2026 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
Expand All @@ -21,6 +21,7 @@ class MemoryOperationsHandler {
virtual ~MemoryOperationsHandler() = default;

virtual MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) = 0;
virtual MemoryOperationsStatus decompress(Device *device, GraphicsAllocation &gfxAllocation) = 0;
virtual MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) = 0;
virtual MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) = 0;
virtual MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) = 0;
Expand Down
7 changes: 4 additions & 3 deletions shared/source/memory_manager/unified_memory_manager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -508,11 +508,11 @@ void *SVMAllocsManager::createUnifiedMemoryAllocation(size_t size,
bool compressionEnabled = false;
AllocationType allocationType = getGraphicsAllocationTypeAndCompressionPreference(memoryProperties, compressionEnabled);

bool preferCompressed = true;
if (compressionEnabled && memoryProperties.device) {
auto *releaseHelper = memoryProperties.device->getReleaseHelper();
if (releaseHelper && !releaseHelper->isUsmCompressionSupportedOnPeerAccess() &&
if (not memoryProperties.device->getExecutionEnvironment()->isResourceDecompressionEnabled() &&
memoryProperties.device->hasAnyPeerAccess().value_or(false)) {
compressionEnabled = false;
preferCompressed = false;
}
}

Expand All @@ -533,6 +533,7 @@ void *SVMAllocsManager::createUnifiedMemoryAllocation(size_t size,
unifiedMemoryProperties.flags.uncacheable = memoryProperties.allocationFlags.flags.locallyUncachedResource;
unifiedMemoryProperties.flags.preferCompressed = compressionEnabled || memoryProperties.allocationFlags.flags.compressedHint;
unifiedMemoryProperties.flags.preferCompressed &= memoryManager->isCompressionSupportedForShareable(memoryProperties.allocationFlags.flags.shareable);
unifiedMemoryProperties.flags.preferCompressed &= preferCompressed;
unifiedMemoryProperties.flags.resource48Bit = memoryProperties.allocationFlags.flags.resource48Bit;

if (memoryProperties.memoryType == InternalMemoryType::deviceUnifiedMemory) {
Expand Down
4 changes: 4 additions & 0 deletions shared/source/os_interface/aub_memory_operations_handler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ MemoryOperationsStatus AubMemoryOperationsHandler::makeResidentWithinDevice(Arra
return MemoryOperationsStatus::success;
}

MemoryOperationsStatus AubMemoryOperationsHandler::decompress(Device *device, GraphicsAllocation &gfxAllocation) {
return MemoryOperationsStatus::unsupported;
}

MemoryOperationsStatus AubMemoryOperationsHandler::lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) {
return makeResident(device, gfxAllocations, false, false);
}
Expand Down
1 change: 1 addition & 0 deletions shared/source/os_interface/aub_memory_operations_handler.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ class AubMemoryOperationsHandler : public MemoryOperationsHandler {
~AubMemoryOperationsHandler() override = default;

MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override;
MemoryOperationsStatus decompress(Device *device, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override;
MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override;
Expand Down
2 changes: 1 addition & 1 deletion shared/source/os_interface/linux/drm_buffer_object.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ void BufferObject::printBOBindingResult(OsContext *osContext, uint32_t vmHandleI
int BufferObject::bind(OsContext *osContext, uint32_t vmHandleId, const bool forcePagingFence) {
int retVal = 0;
auto contextId = getOsContextId(osContext);
if (!this->bindInfo[contextId][vmHandleId]) {
if (!this->bindInfo[contextId][vmHandleId] || this->isResourceDecompressRequired()) {
retVal = this->drm->bindBufferObject(osContext, vmHandleId, this, forcePagingFence);
if (debugManager.flags.PrintBOBindingResult.get()) {
printBOBindingResult(osContext, vmHandleId, true, retVal);
Expand Down
7 changes: 7 additions & 0 deletions shared/source/os_interface/linux/drm_buffer_object.h
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,9 @@ class BufferObject {
bool isReadOnlyGpuResource() {
return readOnlyGpuResource;
}
bool isResourceDecompressRequired() {
return requiresResourceDecompress;
}

void setAsReadOnly(bool isReadOnly) {
readOnlyGpuResource = isReadOnly;
Expand All @@ -184,6 +187,9 @@ class BufferObject {
void requireImmediateBinding(bool required) {
requiresImmediateBinding = required;
}
void requireResourceDecompress(bool required) {
requiresResourceDecompress = required;
}

bool isExplicitResidencyRequired() {
return requiresExplicitResidency;
Expand Down Expand Up @@ -308,6 +314,7 @@ class BufferObject {
bool boHandleShared = false;
bool allowCapture = false;
bool requiresImmediateBinding = false;
bool requiresResourceDecompress = false;
bool requiresExplicitResidency = false;
bool requiresLocked = false;
bool chunked = false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,51 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResident(Device *devi
return result;
}

MemoryOperationsStatus DrmMemoryOperationsHandlerBind::decompress(Device *device, GraphicsAllocation &gfxAllocation) {
auto drmAllocation = static_cast<DrmAllocation *>(&gfxAllocation);
auto &engines = device->getAllEngines();

std::lock_guard<std::mutex> lock(mutex);

auto setDecompressOnBOs = [&](bool value) {
if (drmAllocation->storageInfo.isChunked) {
if (auto bo = drmAllocation->getBO()) {
bo->requireResourceDecompress(value);
}
} else {
for (auto &bo : drmAllocation->getBOs()) {
if (bo) {
bo->requireResourceDecompress(value);
}
}
}
};

for (const auto &engine : engines) {
auto osContext = engine.osContext;
auto deviceBitfield = osContext->getDeviceBitfield();

uint32_t devicesDone = 0u;
for (uint32_t vmHandleId = 0u; devicesDone < deviceBitfield.count(); vmHandleId++) {
if (not deviceBitfield.test(vmHandleId)) {
continue;
}
devicesDone++;

setDecompressOnBOs(true);

int retVal = drmAllocation->bindBOs(osContext, vmHandleId, nullptr, true, false);

setDecompressOnBOs(false);

if (retVal) {
return MemoryOperationsStatus::failed;
}
}
}
return MemoryOperationsStatus::success;
}

MemoryOperationsStatus DrmMemoryOperationsHandlerBind::lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) {
for (auto gfxAllocation = gfxAllocations.begin(); gfxAllocation != gfxAllocations.end(); gfxAllocation++) {
(*gfxAllocation)->setLockedMemory(true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ class DrmMemoryOperationsHandlerBind : public DrmMemoryOperationsHandler {

MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable, const bool forcePagingFence, const bool acquireLock) override;
MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override;
MemoryOperationsStatus decompress(Device *device, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override;
MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::makeResident(Device *d
return ret;
}

MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::decompress(Device *device, GraphicsAllocation &gfxAllocation) {
return MemoryOperationsStatus::unsupported;
}

MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) {
OsContext *osContext = nullptr;
for (auto gfxAllocation = gfxAllocations.begin(); gfxAllocation != gfxAllocations.end(); gfxAllocation++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class DrmMemoryOperationsHandlerDefault : public DrmMemoryOperationsHandler {

MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable, const bool forcePagingFence, const bool acquireLock) override;
MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations, bool isDummyExecNeeded, const bool forcePagingFence) override;
MemoryOperationsStatus decompress(Device *device, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override;
MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override;
Expand Down
Loading