diff --git a/sklearn/datasets/tests/test_base.py b/sklearn/datasets/tests/test_base.py
index 3e27a6119554e..e8840a4d8829f 100644
--- a/sklearn/datasets/tests/test_base.py
+++ b/sklearn/datasets/tests/test_base.py
@@ -341,7 +341,10 @@ def test_load_boston_alternative():
     boston_sklearn = load_boston()
 
     data_url = "http://lib.stat.cmu.edu/datasets/boston"
-    raw_df = pd.read_csv(data_url, sep=r"\s+", skiprows=22, header=None)
+    try:
+        raw_df = pd.read_csv(data_url, sep=r"\s+", skiprows=22, header=None)
+    except ConnectionError as e:
+        pytest.xfail(f"The dataset can't be downloaded. Got exception: {e}")
     data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]])
     target = raw_df.values[1::2, 2]