@@ -1084,6 +1084,80 @@ long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
1084
1084
return lret ;
1085
1085
}
1086
1086
1087
+ /**
1088
+ * xe_bo_notifier_prepare_pinned() - Prepare a pinned VRAM object to be backed
1089
+ * up in system memory.
1090
+ * @bo: The buffer object to prepare.
1091
+ *
1092
+ * On successful completion, the object backup pages are allocated. Expectation
1093
+ * is that this is called from the PM notifier, prior to suspend/hibernation.
1094
+ *
1095
+ * Return: 0 on success. Negative error code on failure.
1096
+ */
1097
+ int xe_bo_notifier_prepare_pinned (struct xe_bo * bo )
1098
+ {
1099
+ struct xe_device * xe = ttm_to_xe_device (bo -> ttm .bdev );
1100
+ struct xe_bo * backup ;
1101
+ int ret = 0 ;
1102
+
1103
+ xe_bo_lock (bo , false);
1104
+
1105
+ xe_assert (xe , !bo -> backup_obj );
1106
+
1107
+ /*
1108
+ * Since this is called from the PM notifier we might have raced with
1109
+ * someone unpinning this after we dropped the pinned list lock and
1110
+ * grabbing the above bo lock.
1111
+ */
1112
+ if (!xe_bo_is_pinned (bo ))
1113
+ goto out_unlock_bo ;
1114
+
1115
+ if (!xe_bo_is_vram (bo ))
1116
+ goto out_unlock_bo ;
1117
+
1118
+ if (bo -> flags & XE_BO_FLAG_PINNED_NORESTORE )
1119
+ goto out_unlock_bo ;
1120
+
1121
+ backup = ___xe_bo_create_locked (xe , NULL , NULL , bo -> ttm .base .resv , NULL , bo -> size ,
1122
+ DRM_XE_GEM_CPU_CACHING_WB , ttm_bo_type_kernel ,
1123
+ XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
1124
+ XE_BO_FLAG_PINNED );
1125
+ if (IS_ERR (backup )) {
1126
+ ret = PTR_ERR (backup );
1127
+ goto out_unlock_bo ;
1128
+ }
1129
+
1130
+ backup -> parent_obj = xe_bo_get (bo ); /* Released by bo_destroy */
1131
+ ttm_bo_pin (& backup -> ttm );
1132
+ bo -> backup_obj = backup ;
1133
+
1134
+ out_unlock_bo :
1135
+ xe_bo_unlock (bo );
1136
+ return ret ;
1137
+ }
1138
+
1139
+ /**
1140
+ * xe_bo_notifier_unprepare_pinned() - Undo the previous prepare operation.
1141
+ * @bo: The buffer object to undo the prepare for.
1142
+ *
1143
+ * Always returns 0. The backup object is removed, if still present. Expectation
1144
+ * it that this called from the PM notifier when undoing the prepare step.
1145
+ *
1146
+ * Return: Always returns 0.
1147
+ */
1148
+ int xe_bo_notifier_unprepare_pinned (struct xe_bo * bo )
1149
+ {
1150
+ xe_bo_lock (bo , false);
1151
+ if (bo -> backup_obj ) {
1152
+ ttm_bo_unpin (& bo -> backup_obj -> ttm );
1153
+ xe_bo_put (bo -> backup_obj );
1154
+ bo -> backup_obj = NULL ;
1155
+ }
1156
+ xe_bo_unlock (bo );
1157
+
1158
+ return 0 ;
1159
+ }
1160
+
1087
1161
/**
1088
1162
* xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory
1089
1163
* @bo: The buffer object to move.
@@ -1098,7 +1172,8 @@ long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
1098
1172
int xe_bo_evict_pinned (struct xe_bo * bo )
1099
1173
{
1100
1174
struct xe_device * xe = ttm_to_xe_device (bo -> ttm .bdev );
1101
- struct xe_bo * backup ;
1175
+ struct xe_bo * backup = bo -> backup_obj ;
1176
+ bool backup_created = false;
1102
1177
bool unmap = false;
1103
1178
int ret = 0 ;
1104
1179
@@ -1120,12 +1195,17 @@ int xe_bo_evict_pinned(struct xe_bo *bo)
1120
1195
if (bo -> flags & XE_BO_FLAG_PINNED_NORESTORE )
1121
1196
goto out_unlock_bo ;
1122
1197
1123
- backup = xe_bo_create_locked (xe , NULL , NULL , bo -> size , ttm_bo_type_kernel ,
1124
- XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
1125
- XE_BO_FLAG_PINNED );
1126
- if (IS_ERR (backup )) {
1127
- ret = PTR_ERR (backup );
1128
- goto out_unlock_bo ;
1198
+ if (!backup ) {
1199
+ backup = ___xe_bo_create_locked (xe , NULL , NULL , bo -> ttm .base .resv , NULL , bo -> size ,
1200
+ DRM_XE_GEM_CPU_CACHING_WB , ttm_bo_type_kernel ,
1201
+ XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
1202
+ XE_BO_FLAG_PINNED );
1203
+ if (IS_ERR (backup )) {
1204
+ ret = PTR_ERR (backup );
1205
+ goto out_unlock_bo ;
1206
+ }
1207
+ backup -> parent_obj = xe_bo_get (bo ); /* Released by bo_destroy */
1208
+ backup_created = true;
1129
1209
}
1130
1210
1131
1211
if (xe_bo_is_user (bo ) || (bo -> flags & XE_BO_FLAG_PINNED_LATE_RESTORE )) {
@@ -1173,12 +1253,12 @@ int xe_bo_evict_pinned(struct xe_bo *bo)
1173
1253
bo -> size );
1174
1254
}
1175
1255
1176
- bo -> backup_obj = backup ;
1256
+ if (!bo -> backup_obj )
1257
+ bo -> backup_obj = backup ;
1177
1258
1178
1259
out_backup :
1179
1260
xe_bo_vunmap (backup );
1180
- xe_bo_unlock (backup );
1181
- if (ret )
1261
+ if (ret && backup_created )
1182
1262
xe_bo_put (backup );
1183
1263
out_unlock_bo :
1184
1264
if (unmap )
@@ -1212,15 +1292,12 @@ int xe_bo_restore_pinned(struct xe_bo *bo)
1212
1292
if (!backup )
1213
1293
return 0 ;
1214
1294
1215
- xe_bo_lock (backup , false);
1216
-
1217
- ret = ttm_bo_validate (& backup -> ttm , & backup -> placement , & ctx );
1218
- if (ret )
1219
- goto out_backup ;
1295
+ xe_bo_lock (bo , false);
1220
1296
1221
- if (WARN_ON (!dma_resv_trylock (bo -> ttm .base .resv ))) {
1222
- ret = - EBUSY ;
1223
- goto out_backup ;
1297
+ if (!xe_bo_is_pinned (backup )) {
1298
+ ret = ttm_bo_validate (& backup -> ttm , & backup -> placement , & ctx );
1299
+ if (ret )
1300
+ goto out_unlock_bo ;
1224
1301
}
1225
1302
1226
1303
if (xe_bo_is_user (bo ) || (bo -> flags & XE_BO_FLAG_PINNED_LATE_RESTORE )) {
@@ -1261,7 +1338,7 @@ int xe_bo_restore_pinned(struct xe_bo *bo)
1261
1338
if (iosys_map_is_null (& bo -> vmap )) {
1262
1339
ret = xe_bo_vmap (bo );
1263
1340
if (ret )
1264
- goto out_unlock_bo ;
1341
+ goto out_backup ;
1265
1342
unmap = true;
1266
1343
}
1267
1344
@@ -1271,15 +1348,17 @@ int xe_bo_restore_pinned(struct xe_bo *bo)
1271
1348
1272
1349
bo -> backup_obj = NULL ;
1273
1350
1351
+ out_backup :
1352
+ xe_bo_vunmap (backup );
1353
+ if (!bo -> backup_obj ) {
1354
+ if (xe_bo_is_pinned (backup ))
1355
+ ttm_bo_unpin (& backup -> ttm );
1356
+ xe_bo_put (backup );
1357
+ }
1274
1358
out_unlock_bo :
1275
1359
if (unmap )
1276
1360
xe_bo_vunmap (bo );
1277
1361
xe_bo_unlock (bo );
1278
- out_backup :
1279
- xe_bo_vunmap (backup );
1280
- xe_bo_unlock (backup );
1281
- if (!bo -> backup_obj )
1282
- xe_bo_put (backup );
1283
1362
return ret ;
1284
1363
}
1285
1364
@@ -1455,16 +1534,22 @@ static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo,
1455
1534
struct xe_res_cursor cursor ;
1456
1535
struct xe_vram_region * vram ;
1457
1536
int bytes_left = len ;
1537
+ int err = 0 ;
1458
1538
1459
1539
xe_bo_assert_held (bo );
1460
1540
xe_device_assert_mem_access (xe );
1461
1541
1462
1542
if (!mem_type_is_vram (ttm_bo -> resource -> mem_type ))
1463
1543
return - EIO ;
1464
1544
1465
- /* FIXME: Use GPU for non-visible VRAM */
1466
- if (!xe_ttm_resource_visible (ttm_bo -> resource ))
1467
- return - EIO ;
1545
+ if (!xe_ttm_resource_visible (ttm_bo -> resource ) || len >= SZ_16K ) {
1546
+ struct xe_migrate * migrate =
1547
+ mem_type_to_migrate (xe , ttm_bo -> resource -> mem_type );
1548
+
1549
+ err = xe_migrate_access_memory (migrate , bo , offset , buf , len ,
1550
+ write );
1551
+ goto out ;
1552
+ }
1468
1553
1469
1554
vram = res_to_mem_region (ttm_bo -> resource );
1470
1555
xe_res_first (ttm_bo -> resource , offset & PAGE_MASK ,
@@ -1488,7 +1573,8 @@ static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo,
1488
1573
xe_res_next (& cursor , PAGE_SIZE );
1489
1574
} while (bytes_left );
1490
1575
1491
- return len ;
1576
+ out :
1577
+ return err ?: len ;
1492
1578
}
1493
1579
1494
1580
const struct ttm_device_funcs xe_ttm_funcs = {
@@ -1532,6 +1618,9 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
1532
1618
if (bo -> vm && xe_bo_is_user (bo ))
1533
1619
xe_vm_put (bo -> vm );
1534
1620
1621
+ if (bo -> parent_obj )
1622
+ xe_bo_put (bo -> parent_obj );
1623
+
1535
1624
mutex_lock (& xe -> mem_access .vram_userfault .lock );
1536
1625
if (!list_empty (& bo -> vram_userfault_link ))
1537
1626
list_del (& bo -> vram_userfault_link );
@@ -2306,6 +2395,13 @@ void xe_bo_unpin(struct xe_bo *bo)
2306
2395
xe_assert (xe , !list_empty (& bo -> pinned_link ));
2307
2396
list_del_init (& bo -> pinned_link );
2308
2397
spin_unlock (& xe -> pinned .lock );
2398
+
2399
+ if (bo -> backup_obj ) {
2400
+ if (xe_bo_is_pinned (bo -> backup_obj ))
2401
+ ttm_bo_unpin (& bo -> backup_obj -> ttm );
2402
+ xe_bo_put (bo -> backup_obj );
2403
+ bo -> backup_obj = NULL ;
2404
+ }
2309
2405
}
2310
2406
ttm_bo_unpin (& bo -> ttm );
2311
2407
if (bo -> ttm .ttm && ttm_tt_is_populated (bo -> ttm .ttm ))
0 commit comments