@@ -48,7 +48,8 @@ struct amdgpu_mn {
48
48
/* protected by adev->mn_lock */
49
49
struct hlist_node node ;
50
50
51
- /* objects protected by mm->mmap_sem */
51
+ /* objects protected by lock */
52
+ struct mutex lock ;
52
53
struct rb_root objects ;
53
54
};
54
55
@@ -72,7 +73,7 @@ static void amdgpu_mn_destroy(struct work_struct *work)
72
73
struct amdgpu_bo * bo , * next_bo ;
73
74
74
75
mutex_lock (& adev -> mn_lock );
75
- down_write (& rmn -> mm -> mmap_sem );
76
+ mutex_lock (& rmn -> lock );
76
77
hash_del (& rmn -> node );
77
78
rbtree_postorder_for_each_entry_safe (node , next_node , & rmn -> objects ,
78
79
it .rb ) {
@@ -82,7 +83,7 @@ static void amdgpu_mn_destroy(struct work_struct *work)
82
83
}
83
84
kfree (node );
84
85
}
85
- up_write (& rmn -> mm -> mmap_sem );
86
+ mutex_unlock (& rmn -> lock );
86
87
mutex_unlock (& adev -> mn_lock );
87
88
mmu_notifier_unregister_no_release (& rmn -> mn , rmn -> mm );
88
89
kfree (rmn );
@@ -104,6 +105,76 @@ static void amdgpu_mn_release(struct mmu_notifier *mn,
104
105
schedule_work (& rmn -> work );
105
106
}
106
107
108
+ /**
109
+ * amdgpu_mn_invalidate_node - unmap all BOs of a node
110
+ *
111
+ * @node: the node with the BOs to unmap
112
+ *
113
+ * We block for all BOs and unmap them by move them
114
+ * into system domain again.
115
+ */
116
+ static void amdgpu_mn_invalidate_node (struct amdgpu_mn_node * node ,
117
+ unsigned long start ,
118
+ unsigned long end )
119
+ {
120
+ struct amdgpu_bo * bo ;
121
+ long r ;
122
+
123
+ list_for_each_entry (bo , & node -> bos , mn_list ) {
124
+
125
+ if (!amdgpu_ttm_tt_affect_userptr (bo -> tbo .ttm , start , end ))
126
+ continue ;
127
+
128
+ r = amdgpu_bo_reserve (bo , true);
129
+ if (r ) {
130
+ DRM_ERROR ("(%ld) failed to reserve user bo\n" , r );
131
+ continue ;
132
+ }
133
+
134
+ r = reservation_object_wait_timeout_rcu (bo -> tbo .resv ,
135
+ true, false, MAX_SCHEDULE_TIMEOUT );
136
+ if (r <= 0 )
137
+ DRM_ERROR ("(%ld) failed to wait for user bo\n" , r );
138
+
139
+ amdgpu_ttm_placement_from_domain (bo , AMDGPU_GEM_DOMAIN_CPU );
140
+ r = ttm_bo_validate (& bo -> tbo , & bo -> placement , false, false);
141
+ if (r )
142
+ DRM_ERROR ("(%ld) failed to validate user bo\n" , r );
143
+
144
+ amdgpu_bo_unreserve (bo );
145
+ }
146
+ }
147
+
148
+ /**
149
+ * amdgpu_mn_invalidate_page - callback to notify about mm change
150
+ *
151
+ * @mn: our notifier
152
+ * @mn: the mm this callback is about
153
+ * @address: address of invalidate page
154
+ *
155
+ * Invalidation of a single page. Blocks for all BOs mapping it
156
+ * and unmap them by move them into system domain again.
157
+ */
158
+ static void amdgpu_mn_invalidate_page (struct mmu_notifier * mn ,
159
+ struct mm_struct * mm ,
160
+ unsigned long address )
161
+ {
162
+ struct amdgpu_mn * rmn = container_of (mn , struct amdgpu_mn , mn );
163
+ struct interval_tree_node * it ;
164
+
165
+ mutex_lock (& rmn -> lock );
166
+
167
+ it = interval_tree_iter_first (& rmn -> objects , address , address );
168
+ if (it ) {
169
+ struct amdgpu_mn_node * node ;
170
+
171
+ node = container_of (it , struct amdgpu_mn_node , it );
172
+ amdgpu_mn_invalidate_node (node , address , address );
173
+ }
174
+
175
+ mutex_unlock (& rmn -> lock );
176
+ }
177
+
107
178
/**
108
179
* amdgpu_mn_invalidate_range_start - callback to notify about mm change
109
180
*
@@ -126,44 +197,24 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
126
197
/* notification is exclusive, but interval is inclusive */
127
198
end -= 1 ;
128
199
200
+ mutex_lock (& rmn -> lock );
201
+
129
202
it = interval_tree_iter_first (& rmn -> objects , start , end );
130
203
while (it ) {
131
204
struct amdgpu_mn_node * node ;
132
- struct amdgpu_bo * bo ;
133
- long r ;
134
205
135
206
node = container_of (it , struct amdgpu_mn_node , it );
136
207
it = interval_tree_iter_next (it , start , end );
137
208
138
- list_for_each_entry (bo , & node -> bos , mn_list ) {
139
-
140
- if (!amdgpu_ttm_tt_affect_userptr (bo -> tbo .ttm , start ,
141
- end ))
142
- continue ;
143
-
144
- r = amdgpu_bo_reserve (bo , true);
145
- if (r ) {
146
- DRM_ERROR ("(%ld) failed to reserve user bo\n" , r );
147
- continue ;
148
- }
149
-
150
- r = reservation_object_wait_timeout_rcu (bo -> tbo .resv ,
151
- true, false, MAX_SCHEDULE_TIMEOUT );
152
- if (r <= 0 )
153
- DRM_ERROR ("(%ld) failed to wait for user bo\n" , r );
154
-
155
- amdgpu_ttm_placement_from_domain (bo , AMDGPU_GEM_DOMAIN_CPU );
156
- r = ttm_bo_validate (& bo -> tbo , & bo -> placement , false, false);
157
- if (r )
158
- DRM_ERROR ("(%ld) failed to validate user bo\n" , r );
159
-
160
- amdgpu_bo_unreserve (bo );
161
- }
209
+ amdgpu_mn_invalidate_node (node , start , end );
162
210
}
211
+
212
+ mutex_unlock (& rmn -> lock );
163
213
}
164
214
165
215
static const struct mmu_notifier_ops amdgpu_mn_ops = {
166
216
.release = amdgpu_mn_release ,
217
+ .invalidate_page = amdgpu_mn_invalidate_page ,
167
218
.invalidate_range_start = amdgpu_mn_invalidate_range_start ,
168
219
};
169
220
@@ -196,6 +247,7 @@ static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
196
247
rmn -> adev = adev ;
197
248
rmn -> mm = mm ;
198
249
rmn -> mn .ops = & amdgpu_mn_ops ;
250
+ mutex_init (& rmn -> lock );
199
251
rmn -> objects = RB_ROOT ;
200
252
201
253
r = __mmu_notifier_register (& rmn -> mn , mm );
@@ -242,7 +294,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
242
294
243
295
INIT_LIST_HEAD (& bos );
244
296
245
- down_write (& rmn -> mm -> mmap_sem );
297
+ mutex_lock (& rmn -> lock );
246
298
247
299
while ((it = interval_tree_iter_first (& rmn -> objects , addr , end ))) {
248
300
kfree (node );
@@ -256,7 +308,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
256
308
if (!node ) {
257
309
node = kmalloc (sizeof (struct amdgpu_mn_node ), GFP_KERNEL );
258
310
if (!node ) {
259
- up_write (& rmn -> mm -> mmap_sem );
311
+ mutex_unlock (& rmn -> lock );
260
312
return - ENOMEM ;
261
313
}
262
314
}
@@ -271,7 +323,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
271
323
272
324
interval_tree_insert (& node -> it , & rmn -> objects );
273
325
274
- up_write (& rmn -> mm -> mmap_sem );
326
+ mutex_unlock (& rmn -> lock );
275
327
276
328
return 0 ;
277
329
}
@@ -297,7 +349,7 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
297
349
return ;
298
350
}
299
351
300
- down_write (& rmn -> mm -> mmap_sem );
352
+ mutex_lock (& rmn -> lock );
301
353
302
354
/* save the next list entry for later */
303
355
head = bo -> mn_list .next ;
@@ -312,6 +364,6 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
312
364
kfree (node );
313
365
}
314
366
315
- up_write (& rmn -> mm -> mmap_sem );
367
+ mutex_unlock (& rmn -> lock );
316
368
mutex_unlock (& adev -> mn_lock );
317
369
}
0 commit comments