@@ -75,6 +75,9 @@ struct kmem_cache *amd_iommu_irq_cache;
75
75
76
76
static void detach_device (struct device * dev );
77
77
78
+ static void set_dte_entry (struct amd_iommu * iommu ,
79
+ struct iommu_dev_data * dev_data );
80
+
78
81
/****************************************************************************
79
82
*
80
83
* Helper functions
@@ -1666,16 +1669,19 @@ static void free_gcr3_tbl_level2(u64 *tbl)
1666
1669
}
1667
1670
}
1668
1671
1669
- static void free_gcr3_table (struct protection_domain * domain )
1672
+ static void free_gcr3_table (struct gcr3_tbl_info * gcr3_info )
1670
1673
{
1671
- if (domain -> glx == 2 )
1672
- free_gcr3_tbl_level2 (domain -> gcr3_tbl );
1673
- else if (domain -> glx == 1 )
1674
- free_gcr3_tbl_level1 (domain -> gcr3_tbl );
1674
+ if (gcr3_info -> glx == 2 )
1675
+ free_gcr3_tbl_level2 (gcr3_info -> gcr3_tbl );
1676
+ else if (gcr3_info -> glx == 1 )
1677
+ free_gcr3_tbl_level1 (gcr3_info -> gcr3_tbl );
1675
1678
else
1676
- BUG_ON (domain -> glx != 0 );
1679
+ WARN_ON_ONCE (gcr3_info -> glx != 0 );
1680
+
1681
+ gcr3_info -> glx = 0 ;
1677
1682
1678
- free_page ((unsigned long )domain -> gcr3_tbl );
1683
+ free_page ((unsigned long )gcr3_info -> gcr3_tbl );
1684
+ gcr3_info -> gcr3_tbl = NULL ;
1679
1685
}
1680
1686
1681
1687
/*
@@ -1694,22 +1700,23 @@ static int get_gcr3_levels(int pasids)
1694
1700
return levels ? (DIV_ROUND_UP (levels , 9 ) - 1 ) : levels ;
1695
1701
}
1696
1702
1697
- /* Note: This function expects iommu_domain->lock to be held prior calling the function. */
1698
- static int setup_gcr3_table ( struct protection_domain * domain , int pasids )
1703
+ static int setup_gcr3_table ( struct gcr3_tbl_info * gcr3_info ,
1704
+ struct amd_iommu * iommu , int pasids )
1699
1705
{
1700
1706
int levels = get_gcr3_levels (pasids );
1707
+ int nid = iommu ? dev_to_node (& iommu -> dev -> dev ) : NUMA_NO_NODE ;
1701
1708
1702
1709
if (levels > amd_iommu_max_glx_val )
1703
1710
return - EINVAL ;
1704
1711
1705
- domain -> gcr3_tbl = alloc_pgtable_page (domain -> nid , GFP_ATOMIC );
1706
- if (domain -> gcr3_tbl == NULL )
1707
- return - ENOMEM ;
1712
+ if (gcr3_info -> gcr3_tbl )
1713
+ return - EBUSY ;
1708
1714
1709
- domain -> glx = levels ;
1710
- domain -> flags |= PD_IOMMUV2_MASK ;
1715
+ gcr3_info -> gcr3_tbl = alloc_pgtable_page (nid , GFP_KERNEL );
1716
+ if (gcr3_info -> gcr3_tbl == NULL )
1717
+ return - ENOMEM ;
1711
1718
1712
- amd_iommu_domain_update ( domain ) ;
1719
+ gcr3_info -> glx = levels ;
1713
1720
1714
1721
return 0 ;
1715
1722
}
@@ -1808,6 +1815,7 @@ static void set_dte_entry(struct amd_iommu *iommu,
1808
1815
u16 devid = dev_data -> devid ;
1809
1816
struct protection_domain * domain = dev_data -> domain ;
1810
1817
struct dev_table_entry * dev_table = get_dev_table (iommu );
1818
+ struct gcr3_tbl_info * gcr3_info = & dev_data -> gcr3_info ;
1811
1819
1812
1820
if (domain -> iop .mode != PAGE_MODE_NONE )
1813
1821
pte_root = iommu_virt_to_phys (domain -> iop .root );
@@ -1835,9 +1843,9 @@ static void set_dte_entry(struct amd_iommu *iommu,
1835
1843
if (domain -> dirty_tracking )
1836
1844
pte_root |= DTE_FLAG_HAD ;
1837
1845
1838
- if (domain -> flags & PD_IOMMUV2_MASK ) {
1839
- u64 gcr3 = iommu_virt_to_phys (domain -> gcr3_tbl );
1840
- u64 glx = domain -> glx ;
1846
+ if (gcr3_info && gcr3_info -> gcr3_tbl ) {
1847
+ u64 gcr3 = iommu_virt_to_phys (gcr3_info -> gcr3_tbl );
1848
+ u64 glx = gcr3_info -> glx ;
1841
1849
u64 tmp ;
1842
1850
1843
1851
pte_root |= DTE_FLAG_GV ;
@@ -1865,7 +1873,8 @@ static void set_dte_entry(struct amd_iommu *iommu,
1865
1873
((u64 )GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT );
1866
1874
}
1867
1875
1868
- if (domain -> flags & PD_GIOV_MASK )
1876
+ /* GIOV is supported with V2 page table mode only */
1877
+ if (pdom_is_v2_pgtbl_mode (domain ))
1869
1878
pte_root |= DTE_FLAG_GIOV ;
1870
1879
}
1871
1880
@@ -1922,14 +1931,14 @@ static int do_attach(struct iommu_dev_data *dev_data,
1922
1931
/* Init GCR3 table and update device table */
1923
1932
if (domain -> pd_mode == PD_MODE_V2 ) {
1924
1933
/* By default, setup GCR3 table to support single PASID */
1925
- ret = setup_gcr3_table (dev_data -> domain , 1 );
1934
+ ret = setup_gcr3_table (& dev_data -> gcr3_info , iommu , 1 );
1926
1935
if (ret )
1927
1936
return ret ;
1928
1937
1929
1938
ret = update_gcr3 (dev_data , 0 ,
1930
1939
iommu_virt_to_phys (domain -> iop .pgd ), true);
1931
1940
if (ret ) {
1932
- free_gcr3_table (dev_data -> domain );
1941
+ free_gcr3_table (& dev_data -> gcr3_info );
1933
1942
return ret ;
1934
1943
}
1935
1944
}
@@ -1951,7 +1960,7 @@ static void do_detach(struct iommu_dev_data *dev_data)
1951
1960
/* Clear GCR3 table */
1952
1961
if (domain -> pd_mode == PD_MODE_V2 ) {
1953
1962
update_gcr3 (dev_data , 0 , 0 , false);
1954
- free_gcr3_table (dev_data -> domain );
1963
+ free_gcr3_table (& dev_data -> gcr3_info );
1955
1964
}
1956
1965
1957
1966
/* Update data structures */
0 commit comments