diff --git a/.circleci/config.yml b/.circleci/config.yml index a8df5dab..716a2610 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: build-docs: working_directory: ~/repo docker: - - image: cimg/python:3.8 + - image: cimg/python:3.10 steps: - checkout @@ -49,7 +49,7 @@ jobs: deploy-docs: working_directory: ~/repo docker: - - image: circleci/python:3.8.5-buster + - image: cimg/python:3.10 steps: - checkout @@ -70,7 +70,7 @@ jobs: - add_ssh_keys: fingerprints: - db:84:df:44:ad:77:d0:aa:2d:81:c9:73:30:9d:21:37 + 5c:54:62:37:75:7f:4d:14:f4:07:82:1c:50:0d:ee:9b - run: name: deploy to gh-pages diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..8db2976b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: ".github/workflows" + schedule: + interval: "monthly" + groups: + actions: + patterns: + - "*" + labels: + - "infrastructure" diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml new file mode 100644 index 00000000..7b54b916 --- /dev/null +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -0,0 +1,59 @@ +name: Test notebooks + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '0 5 * * 1' + workflow_dispatch: + +jobs: + tests: + name: ${{ matrix.os }} ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Run all supported OS for one Python version, then add a few extra scenarios + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ['3.10'] + toxenv: [py310-test] + name: ['with Python 3.10',] + + include: + - python-version: '3.9' + toxenv: py39-test-oldestdeps + name: with Python 3.9 and oldest versioned dependencies + os: ubuntu-latest + + - python-version: '3.11' + toxenv: py311-test + name: with Python 3.11 and latest released version of dependencies + os: ubuntu-latest + + - python-version: '3.12' + toxenv: py312-test-predeps + name: with Python 3.12 and latest or pre-release version of dependencies + os: ubuntu-latest + + - python-version: '3.12' + toxenv: py312-test-devdeps + name: with Python 3.12 and developer versioned dependencies + os: ubuntu-latest + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 #v5.6.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: python -m pip install --upgrade tox + + - name: Test with nbval + run: tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }} diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index 5115024b..88ac6e4f 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -5,8 +5,9 @@ jobs: name: Run CircleCI artifacts redirector steps: - name: GitHub Action step - uses: larsoner/circleci-artifacts-redirector-action@master + uses: scientific-python/circleci-artifacts-redirector-action@4e13a10d89177f4bfc8007a7064bdbeda848d8d1 # v1.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} + api-token: ${{ secrets.CIRCLE_TOKEN }} artifact-path: 0/site/_build/html/index.html circleci-jobs: build-docs diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index c58fbc9a..f4dce2f5 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -1,4 +1,4 @@ -name: Build site +name: Test building site with conda environment on: push: @@ -7,6 +7,9 @@ on: pull_request: branches: - main + schedule: + - cron: '0 5 * * 1' + workflow_dispatch: jobs: test: @@ -21,16 +24,15 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 with: auto-update-conda: true activate-environment: numpy-tutorials environment-file: environment.yml - miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true - python-version: "3.10" + python-version: "3.11" auto-activate-base: false - name: inspect and build id: build_step @@ -40,7 +42,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: sphinx-build-artifact path: site/_build/html/reports @@ -48,4 +50,3 @@ jobs: - name: fail on build errors if: steps.build_step.outcome != 'success' run: exit 1 - diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml deleted file mode 100644 index 9489bff9..00000000 --- a/.github/workflows/notebooks.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Test notebooks - -on: - push: - branches: - - main - pull_request: - branches: - - main - schedule: - - cron: '0 5 * * 1' - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - max-parallel: 12 - matrix: - os: [Ubuntu-20.04, macOS-latest] - python-version: [3.8, 3.9, "3.10"] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install -r site/requirements.txt -r requirements.txt - python -m pip list - - - name: Test with nbval - run: | - python -m pip install pytest nbval - find content/ -name "*.md" -exec jupytext --to notebook {} \; - # TODO: find better way to exclude notebooks from test - rm content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.ipynb - rm content/pairing.ipynb - rm content/tutorial-style-guide.ipynb - rm content/tutorial-nlp-from-scratch.ipynb - # Test notebook execution - pytest --nbval-lax --durations=10 content/ diff --git a/.gitignore b/.gitignore index fe8151b1..bee3f8ec 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,7 @@ site/notebooks/* content/mooreslaw_regression* content/tutorial-x-ray-image-processing/xray_image.gif content/video +content/*ipynb +content/tutorial-nlp-from-scratch/parameters.npy +content/tutorial-nlp-from-scratch/*ipynb +content/x_y-squared* diff --git a/LICENSE.txt b/LICENSE.txt index 8ce64521..014d51c9 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2005-2020, NumPy Developers. +Copyright (c) 2005-2023, NumPy Developers. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/content/mooreslaw-tutorial.md b/content/mooreslaw-tutorial.md index 18a13dcc..42c0de15 100644 --- a/content/mooreslaw-tutorial.md +++ b/content/mooreslaw-tutorial.md @@ -21,8 +21,7 @@ _The number of transistors reported per a given chip plotted on a log scale in t In 1965, engineer Gordon Moore [predicted](https://en.wikipedia.org/wiki/Moore%27s_law) that transistors on a chip would double every two years in the coming decade -[[1](https://en.wikipedia.org/wiki/Moore%27s_law), -[2](https://newsroom.intel.com/wp-content/uploads/sites/11/2018/05/moores-law-electronics.pdf)]. +[[1](https://en.wikipedia.org/wiki/Moore%27s_law)]. You'll compare Moore's prediction against actual transistor counts in the 53 years following his prediction. You will determine the best-fit constants to describe the exponential growth of transistors on semiconductors compared to Moore's Law. @@ -44,19 +43,17 @@ the 53 years following his prediction. You will determine the best-fit constants * NumPy * [Matplotlib](https://matplotlib.org/) -* [statsmodels](https://www.statsmodels.org) ordinary linear regression imported with the following commands ```{code-cell} import matplotlib.pyplot as plt import numpy as np -import statsmodels.api as sm ``` **2.** Since this is an exponential growth law you need a little background in doing math with [natural logs](https://en.wikipedia.org/wiki/Natural_logarithm) and [exponentials](https://en.wikipedia.org/wiki/Exponential_function). -You'll use these NumPy, Matplotlib, and statsmodels functions: +You'll use these NumPy and Matplotlib functions: * [`np.loadtxt`](https://numpy.org/doc/stable/reference/generated/numpy.loadtxt.html): this function loads text into a NumPy array * [`np.log`](https://numpy.org/doc/stable/reference/generated/numpy.log.html): this function takes the natural log of all elements in a NumPy array @@ -64,7 +61,6 @@ You'll use these NumPy, Matplotlib, and statsmodels functions: * [`lambda`](https://docs.python.org/3/library/ast.html?highlight=lambda#ast.Lambda): this is a minimal function definition for creating a function model * [`plt.semilogy`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.semilogy.html): this function will plot x-y data onto a figure with a linear x-axis and $\log_{10}$ y-axis [`plt.plot`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.plot.html): this function will plot x-y data on linear axes -* [`sm.OLS`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html): find fitting parameters and standard errors using the statsmodels ordinary least squares model * slicing arrays: view parts of the data loaded into the workspace, slice the arrays e.g. `x[:10]` for the first 10 values in the array, `x` * boolean array indexing: to view parts of the data that match a given condition use boolean operations to index an array * [`np.block`](https://numpy.org/doc/stable/reference/generated/numpy.block.html): to combine arrays into 2D arrays @@ -133,7 +129,7 @@ print("This is x{:.2f} more transistors than 1971".format(ML_1973 / ML_1971)) Now, make a prediction based upon the historical data for semiconductors per chip. The [Transistor Count -\[4\]](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) +\[3\]](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) each year is in the `transistor_data.csv` file. Before loading a \*.csv file into a NumPy array, its a good idea to inspect the structure of the file first. Then, locate the columns of interest and save them to a @@ -186,7 +182,7 @@ print("trans. cnt:\t", transistor_count[:10]) You are creating a function that predicts the transistor count given a year. You have an _independent variable_, `year`, and a _dependent -variable_, `transistor_count`. Transform the independent variable to +variable_, `transistor_count`. Transform the dependent variable to log-scale, $y_i = \log($ `transistor_count[i]` $),$ @@ -215,59 +211,31 @@ where $\mathbf{y}$ are the observations of the log of the number of transistors in a 1D array and $\mathbf{Z}=[\text{year}_i^1,~\text{year}_i^0]$ are the polynomial terms for $\text{year}_i$ in the first and second columns. By creating this set of regressors in the $\mathbf{Z}-$matrix you set -up an ordinary least squares statistical model. Some clever -NumPy array features will build $\mathbf{Z}$ +up an ordinary least squares statistical model. -1. `year[:,np.newaxis]` : takes the 1D array with shape `(179,)` and turns it into a 2D column vector with shape `(179,1)` -2. `**[1, 0]` : stacks two columns, in the first column is `year**1` and the second column is `year**0 == 1` +`Z` is a linear model with two parameters, i.e. a polynomial with degree `1`. +Therefore we can represent the model with `numpy.polynomial.Polynomial` and +use the fitting functionality to determine the model parameters: ```{code-cell} -Z = year[:, np.newaxis] ** [1, 0] +model = np.polynomial.Polynomial.fit(year, yi, deg=1) ``` -Now that you have the created a matrix of regressors, $\mathbf{Z},$ and -the observations are in vector, $\mathbf{y},$ you can use these -variables to build the an ordinary least squares model with -[`sm.OLS`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html). +By default, `Polynomial.fit` performs the fit in the domain determined by the +independent variable (`year` in this case). +The coefficients for the unscaled and unshifted model can be recovered with the +`convert` method: -```{code-cell} -model = sm.OLS(yi, Z) -``` - -Now, you can view the fitting constants, $A$ and $B$, and their standard -errors. Run the -[`fit`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html) and print the -[`summary`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.RegressionResults.summary.html) to view results as such, ```{code-cell} -results = model.fit() -print(results.summary()) +model = model.convert() +model ``` -The __OLS Regression Results__ summary gives a lot of information about -the regressors, $\mathbf{Z},$ and observations, $\mathbf{y}.$ The most -important outputs for your current analysis are - -``` -================================= - coef std err ---------------------------------- -x1 0.3416 0.006 -const -666.3264 11.890 -================================= -``` -where `x1` is slope, $A=0.3416$, `const` is the intercept, -$B=-666.364$, and `std error` gives the precision of constants -$A=0.342\pm 0.006~\dfrac{\log(\text{transistors}/\text{chip})}{\text{years}}$ and $B=-666\pm -12~\log(\text{transistors}/\text{chip}),$ where the units are in -$\log(\text{transistors}/\text{chip})$. You created an exponential growth model. -To get the constants, save them to an array `AB` with -`results.params` and assign $A$ and $B$ to `x1` and `constant`. +The individual parameters $A$ and $B$ are the coefficients of our linear model: ```{code-cell} -AB = results.params -A = AB[0] -B = AB[1] +B, A = model ``` Did manufacturers double the transistor count every two years? You have @@ -277,24 +245,14 @@ $\dfrac{\text{transistor_count}(\text{year} +2)}{\text{transistor_count}(\text{y \dfrac{e^{B}e^{A( \text{year} + 2)}}{e^{B}e^{A \text{year}}} = e^{2A}$ where increase in number of transistors is $xFactor,$ number of years is -2, and $A$ is the best fit slope on the semilog function. The error in -your -prediction, $\Delta(xFactor),$ comes from the precision of your constant -$A,$ which you calculated as the standard error $\Delta A= 0.006$. - -$\Delta (xFactor) = \frac{\partial}{\partial A}(e^{2A})\Delta A = 2Ae^{2A}\Delta A$ +2, and $A$ is the best fit slope on the semilog function. ```{code-cell} -print("Rate of semiconductors added on a chip every 2 years:") -print( - "\tx{:.2f} +/- {:.2f} semiconductors per chip".format( - np.exp((A) * 2), 2 * A * np.exp(2 * A) * 0.006 - ) -) +print(f"Rate of semiconductors added on a chip every 2 years: {np.exp(2 * A):.2f}") ``` Based upon your least-squares regression model, the number of -semiconductors per chip increased by a factor of $1.98\pm 0.01$ every two +semiconductors per chip increased by a factor of $1.98$ every two years. You have a model that predicts the number of semiconductors each year. Now compare your model to the actual manufacturing reports. Plot the linear regression results and all of the transistor counts. @@ -455,7 +413,7 @@ np.savez( transistor_count=transistor_count, transistor_count_predicted=transistor_count_predicted, transistor_Moores_law=transistor_Moores_law, - regression_csts=AB, + regression_csts=(A, B), ) ``` @@ -561,7 +519,7 @@ double every two years from 1965 through 1975, but the average growth has maintained a consistent increase of $\times 1.98 \pm 0.01$ every two years from 1971 through 2019. In 2015, Moore revised his prediction to say Moore's law should hold until 2025. -[[3](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress)]. +[[2](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress)]. You can share these results as a zipped NumPy array file, `mooreslaw_regression.npz`, or as another csv, `mooreslaw_regression.csv`. The amazing progress in semiconductor @@ -574,6 +532,5 @@ has been over the last half-century. ## References 1. ["Moore's Law." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Moore%27s_law) -2. [Moore, Gordon E. (1965-04-19). "Cramming more components onto integrated circuits". intel.com. Electronics Magazine. Retrieved April 1, 2020.](https://newsroom.intel.com/wp-content/uploads/sites/11/2018/05/moores-law-electronics.pdf) -3. [Courtland, Rachel. "Gordon Moore: The Man Whose Name Means Progress." IEEE Spectrum. 30 Mar. 2015.](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress). -4. ["Transistor Count." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) +2. [Courtland, Rachel. "Gordon Moore: The Man Whose Name Means Progress." IEEE Spectrum. 30 Mar. 2015.](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress). +3. ["Transistor Count." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) diff --git a/content/save-load-arrays.md b/content/save-load-arrays.md index 1960d0de..2620bfaf 100644 --- a/content/save-load-arrays.md +++ b/content/save-load-arrays.md @@ -68,7 +68,7 @@ will assign `x` to the integers from 0 to 9 using [`np.arange`](https://numpy.org/doc/stable/reference/generated/numpy.arange.html). ```{code-cell} -x = np.arange(0, 10, 1) +x = np.arange(10) y = x ** 2 print(x) print(y) @@ -127,7 +127,7 @@ print(load_xy.files) ``` ```{code-cell} -whos +%whos ``` ## Reassign the NpzFile arrays to `x` and `y` @@ -187,18 +187,8 @@ np.savetxt("x_y-squared.csv", X=array_out, header="x, y", delimiter=",") Open the file, `x_y-squared.csv`, and you'll see the following: -``` -# x, y -0.000000000000000000e+00,0.000000000000000000e+00 -1.000000000000000000e+00,1.000000000000000000e+00 -2.000000000000000000e+00,4.000000000000000000e+00 -3.000000000000000000e+00,9.000000000000000000e+00 -4.000000000000000000e+00,1.600000000000000000e+01 -5.000000000000000000e+00,2.500000000000000000e+01 -6.000000000000000000e+00,3.600000000000000000e+01 -7.000000000000000000e+00,4.900000000000000000e+01 -8.000000000000000000e+00,6.400000000000000000e+01 -9.000000000000000000e+00,8.100000000000000000e+01 +```{code-cell} +!head x_y-squared.csv ``` ## Our arrays as a csv file diff --git a/content/tutorial-air-quality-analysis.md b/content/tutorial-air-quality-analysis.md index 14e46acf..fda0dbf3 100644 --- a/content/tutorial-air-quality-analysis.md +++ b/content/tutorial-air-quality-analysis.md @@ -97,7 +97,7 @@ With this, we have successfully imported the data and checked that it is complet ## Calculating the Air Quality Index -We will calculate the AQI using [the method](https://app.cpcbccr.com/ccr_docs/FINAL-REPORT_AQI_.pdf) adopted by the [Central Pollution Control Board](https://www.cpcb.nic.in/national-air-quality-index) of India. To summarize the steps: +We will calculate the AQI using [the method](https://app.cpcbccr.com/ccr_docs/FINAL-REPORT_AQI_.pdf) adopted by the [Central Pollution Control Board](https://www.cpcb.nic.in/national-air-quality-index/) of India. To summarize the steps: - Collect 24-hourly average concentration values for the standard pollutants; 8-hourly in case of CO and O3. diff --git a/content/tutorial-deep-learning-on-mnist.md b/content/tutorial-deep-learning-on-mnist.md index 82aea978..89647016 100644 --- a/content/tutorial-deep-learning-on-mnist.md +++ b/content/tutorial-deep-learning-on-mnist.md @@ -33,7 +33,7 @@ This tutorial was adapted from the work by [Andrew Trask](https://github.com/iam The reader should have some knowledge of Python, NumPy array manipulation, and linear algebra. In addition, you should be familiar with main concepts of [deep learning](https://en.wikipedia.org/wiki/Deep_learning). -To refresh the memory, you can take the [Python](https://docs.python.org/dev/tutorial/index.html) and [Linear algebra on n-dimensional arrays](https://numpy.org/doc/stable/user/tutorial-svd.html) tutorials. +To refresh the memory, you can take the [Python](https://docs.python.org/dev/tutorial/index.html) and [Linear algebra on n-dimensional arrays](https://numpy.org/numpy-tutorials/content/tutorial-svd.html) tutorials. You are advised to read the [Deep learning](http://www.cs.toronto.edu/~hinton/absps/NatureDeepReview.pdf) paper published in 2015 by Yann LeCun, Yoshua Bengio, and Geoffrey Hinton, who are regarded as some of the pioneers of the field. You should also consider reading Andrew Trask's [Grokking Deep Learning](https://www.manning.com/books/grokking-deep-learning), which teaches deep learning with NumPy. @@ -62,7 +62,7 @@ This tutorial can be run locally in an isolated environment, such as [Virtualenv ## 1. Load the MNIST dataset -In this section, you will download the zipped MNIST dataset files originally stored in [Yann LeCun's website](http://yann.lecun.com/exdb/mnist/). Then, you will transform them into 4 files of NumPy array type using built-in Python modules. Finally, you will split the arrays into training and test sets. +In this section, you will download the zipped MNIST dataset files originally developed by Yann LeCun's research team. (More details of the MNIST dataset are available on [Kaggle](https://www.kaggle.com/datasets/hojjatk/mnist-dataset).) Then, you will transform them into 4 files of NumPy array type using built-in Python modules. Finally, you will split the arrays into training and test sets. **1.** Define a variable to store the training/test image/label names of the MNIST dataset in a list: @@ -544,17 +544,13 @@ for j in range(epochs): # Summarize error and accuracy metrics at each epoch print( - "\n" - + "Epoch: " - + str(j) - + " Training set error:" - + str(training_loss / float(len(training_images)))[0:5] - + " Training set accuracy:" - + str(training_accurate_predictions / float(len(training_images))) - + " Test set error:" - + str(test_loss / float(len(test_images)))[0:5] - + " Test set accuracy:" - + str(test_accurate_predictions / float(len(test_images))) + ( + f"Epoch: {j}\n" + f" Training set error: {training_loss / len(training_images):.3f}\n" + f" Training set accuracy: {training_accurate_predictions / len(training_images)}\n" + f" Test set error: {test_loss / len(test_images):.3f}\n" + f" Test set accuracy: {test_accurate_predictions / len(test_images)}" + ) ) ``` @@ -565,39 +561,31 @@ The training process may take many minutes, depending on a number of factors, su After executing the cell above, you can visualize the training and test set errors and accuracy for an instance of this training process. ```{code-cell} +epoch_range = np.arange(epochs) + 1 # Starting from 1 + # The training set metrics. -y_training_error = [ - store_training_loss[i] / float(len(training_images)) - for i in range(len(store_training_loss)) -] -x_training_error = range(1, len(store_training_loss) + 1) -y_training_accuracy = [ - store_training_accurate_pred[i] / float(len(training_images)) - for i in range(len(store_training_accurate_pred)) -] -x_training_accuracy = range(1, len(store_training_accurate_pred) + 1) +training_metrics = { + "accuracy": np.asarray(store_training_accurate_pred) / len(training_images), + "error": np.asarray(store_training_loss) / len(training_images), +} # The test set metrics. -y_test_error = [ - store_test_loss[i] / float(len(test_images)) for i in range(len(store_test_loss)) -] -x_test_error = range(1, len(store_test_loss) + 1) -y_test_accuracy = [ - store_training_accurate_pred[i] / float(len(training_images)) - for i in range(len(store_training_accurate_pred)) -] -x_test_accuracy = range(1, len(store_test_accurate_pred) + 1) +test_metrics = { + "accuracy": np.asarray(store_test_accurate_pred) / len(test_images), + "error": np.asarray(store_test_loss) / len(test_images), +} # Display the plots. fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(15, 5)) -axes[0].set_title("Training set error, accuracy") -axes[0].plot(x_training_accuracy, y_training_accuracy, label="Training set accuracy") -axes[0].plot(x_training_error, y_training_error, label="Training set error") -axes[0].set_xlabel("Epochs") -axes[1].set_title("Test set error, accuracy") -axes[1].plot(x_test_accuracy, y_test_accuracy, label="Test set accuracy") -axes[1].plot(x_test_error, y_test_error, label="Test set error") -axes[1].set_xlabel("Epochs") +for ax, metrics, title in zip( + axes, (training_metrics, test_metrics), ("Training set", "Test set") +): + # Plot the metrics + for metric, values in metrics.items(): + ax.plot(epoch_range, values, label=metric.capitalize()) + ax.set_title(title) + ax.set_xlabel("Epochs") + ax.legend() plt.show() ``` diff --git a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md index 3ef3de03..1598e572 100644 --- a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md +++ b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md @@ -14,7 +14,15 @@ kernelspec: # Deep reinforcement learning with Pong from pixels -This tutorial demonstrates how to implement a deep reinforcement learning (RL) agent from scratch using a policy gradient method that learns to play the [Pong](https://gym.openai.com/envs/Pong-v0/) video game using screen pixels as inputs with NumPy. Your Pong agent will obtain experience on the go using an [artificial neural network](https://en.wikipedia.org/wiki/Artificial_neural_network) as its [policy](https://en.wikipedia.org/wiki/Reinforcement_learning). +```{caution} + +This article is not currently tested due to licensing/installation issues with +the underlying `gym` and `atari-py` dependencies. +Help improve this article by developing an example with reduced dependency +footprint! +``` + +This tutorial demonstrates how to implement a deep reinforcement learning (RL) agent from scratch using a policy gradient method that learns to play the [Pong](https://en.wikipedia.org/wiki/Pong) video game using screen pixels as inputs with NumPy. Your Pong agent will obtain experience on the go using an [artificial neural network](https://en.wikipedia.org/wiki/Artificial_neural_network) as its [policy](https://en.wikipedia.org/wiki/Reinforcement_learning). Pong is a 2D game from 1972 where two players use "rackets" to play a form of table tennis. Each player moves the racket up and down the screen and tries to hit a ball in their opponent's direction by touching it. The goal is to hit the ball such that it goes past the opponent's racket (they miss their shot). According to the rules, if a player reaches 21 points, they win. In Pong, the RL agent that learns to play against an opponent is displayed on the right. @@ -24,7 +32,7 @@ This example is based on the [code](https://gist.github.com/karpathy/a4166c7fe25 ## Prerequisites -- **OpenAI Gym**: To help with the game environment, you will use [Gym](https://gym.openai.com) — an open-source Python interface [developed by OpenAI](https://arxiv.org/abs/1606.01540) that helps perform RL tasks while supporting many simulation environments. +- **OpenAI Gym**: To help with the game environment, you will use [Gym](https://github.com/openai/gym) — an open-source Python interface [developed by OpenAI](https://arxiv.org/abs/1606.01540) that helps perform RL tasks while supporting many simulation environments. - **Python and NumPy**: The reader should have some knowledge of Python, NumPy array manipulation, and linear algebra. - **Deep learning and deep RL**: You should be familiar with main concepts of [deep learning](https://en.wikipedia.org/wiki/Deep_learning), which are explained in the [Deep learning](http://www.cs.toronto.edu/~hinton/absps/NatureDeepReview.pdf) paper published in 2015 by Yann LeCun, Yoshua Bengio, and Geoffrey Hinton, who are regarded as some of the pioneers of the field. The tutorial will try to guide you through the main concepts of deep RL and you will find various literature with links to original sources for your convenience. - **Jupyter notebook environments**: Because RL experiments can require high computing power, you can run the tutorial on the cloud for free using [Binder](https://mybinder.org) or [Google Colaboratory](https://colab.research.google.com/notebooks/intro.ipynb) (which offers free limited GPU and TPU acceleration). diff --git a/content/tutorial-ma.md b/content/tutorial-ma.md index f4aa9a98..fcb69eb2 100644 --- a/content/tutorial-ma.md +++ b/content/tutorial-ma.md @@ -92,7 +92,7 @@ rows of this file, since they contain other data we are not interested in. Separ # Read just the dates for columns 4-18 from the first row dates = np.genfromtxt( filename, - dtype=np.unicode_, + dtype=np.str_, delimiter=",", max_rows=1, usecols=range(4, 18), @@ -102,7 +102,7 @@ dates = np.genfromtxt( # columns, skipping the first six rows locations = np.genfromtxt( filename, - dtype=np.unicode_, + dtype=np.str_, delimiter=",", skip_header=6, usecols=(0, 1), @@ -119,7 +119,7 @@ nbcases = np.genfromtxt( ) ``` -Included in the `numpy.genfromtxt` function call, we have selected the [numpy.dtype](https://numpy.org/devdocs/reference/generated/numpy.dtype.html#numpy.dtype) for each subset of the data (either an integer - `numpy.int_` - or a string of characters - `numpy.unicode_`). We have also used the `encoding` argument to select `utf-8-sig` as the encoding for the file (read more about encoding in the [official Python documentation](https://docs.python.org/3/library/codecs.html#encodings-and-unicode). You can read more about the `numpy.genfromtxt` function from the [Reference Documentation](https://numpy.org/devdocs/reference/generated/numpy.genfromtxt.html#numpy.genfromtxt) or from the [Basic IO tutorial](https://numpy.org/devdocs/user/basics.io.genfromtxt.html). +Included in the `numpy.genfromtxt` function call, we have selected the [numpy.dtype](https://numpy.org/devdocs/reference/generated/numpy.dtype.html#numpy.dtype) for each subset of the data (either an integer - `numpy.int_` - or a string of characters - `numpy.str_`). We have also used the `encoding` argument to select `utf-8-sig` as the encoding for the file (read more about encoding in the [official Python documentation](https://docs.python.org/3/library/codecs.html#encodings-and-unicode). You can read more about the `numpy.genfromtxt` function from the [Reference Documentation](https://numpy.org/devdocs/reference/generated/numpy.genfromtxt.html#numpy.genfromtxt) or from the [Basic IO tutorial](https://numpy.org/devdocs/user/basics.io.genfromtxt.html). +++ @@ -264,14 +264,15 @@ Now, if we want to create a very simple approximation for this data, we should t dates[~china_total.mask] ``` -Finally, we can use the [numpy.polyfit](https://numpy.org/devdocs/reference/generated/numpy.polyfit.html#numpy.polyfit) and [numpy.polyval](https://numpy.org/devdocs/reference/generated/numpy.polyval.html#numpy.polyval) functions to create a cubic polynomial that fits the data as best as possible: +Finally, we can use the +[fitting functionality of the numpy.polynomial](https://numpy.org/doc/stable/reference/generated/numpy.polynomial.polynomial.Polynomial.fit.html) +package to create a cubic polynomial model that fits the data as best as possible: ```{code-cell} t = np.arange(len(china_total)) -params = np.polyfit(t[~china_total.mask], valid, 3) -cubic_fit = np.polyval(params, t) +model = np.polynomial.Polynomial.fit(t[~china_total.mask], valid, deg=3) plt.plot(t, china_total) -plt.plot(t, cubic_fit, "--") +plt.plot(t, model(t), "--") ``` This plot is not so readable since the lines seem to be over each other, so let's summarize in a more elaborate plot. We'll plot the real data when @@ -279,10 +280,10 @@ available, and show the cubic fit for unavailable data, using this fit to comput ```{code-cell} plt.plot(t, china_total) -plt.plot(t[china_total.mask], cubic_fit[china_total.mask], "--", color="orange") -plt.plot(7, np.polyval(params, 7), "r*") +plt.plot(t[china_total.mask], model(t)[china_total.mask], "--", color="orange") +plt.plot(7, model(7), "r*") plt.xticks([0, 7, 13], dates[[0, 7, 13]]) -plt.yticks([0, np.polyval(params, 7), 10000, 17500]) +plt.yticks([0, model(7), 10000, 17500]) plt.legend(["Mainland China", "Cubic estimate", "7 days after start"]) plt.title( "COVID-19 cumulative cases from Jan 21 to Feb 3 2020 - Mainland China\n" diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index 865fd1c9..a4771883 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -15,6 +15,12 @@ jupyter: # Sentiment Analysis on notable speeches of the last decade +```{caution} + +This article is not currently tested. Help improve this tutorial by making it +fully executable! +``` + This tutorial demonstrates how to build a simple Long Short Term memory network (LSTM) from scratch in NumPy to perform sentiment analysis on a socially relevant and ethically acquired dataset. Your deep learning model (the LSTM) is a form of a Recurrent Neural Network and will learn to classify a piece of text as positive or negative from the IMDB reviews dataset. The dataset contains 50,000 movie reviews and corresponding labels. Based on the numeric representations of these reviews and their corresponding labels (supervised learning) the neural network will be trained to learn the sentiment using forward propagation and backpropagation through time since we are dealing with sequential data here. The output will be a vector containing the probabilities that the text samples are positive. @@ -25,7 +31,7 @@ Today, Deep Learning is getting adopted in everyday life and now it is more impo ## Prerequisites -You are expected to be familiar with the Python programming language and array manipulation with NumPy. In addition, some understanding of Linear Algebra and Calculus is recommended. You should also be familiar with how Neural Networks work. For reference, you can visit the [Python](https://docs.python.org/dev/tutorial/index.html), [Linear algebra on n-dimensional arrays](https://numpy.org/doc/stable/user/tutorial-svd.html) and [Calculus](https://d2l.ai/chapter_appendix-mathematics-for-deep-learning/multivariable-calculus.html) tutorials. +You are expected to be familiar with the Python programming language and array manipulation with NumPy. In addition, some understanding of Linear Algebra and Calculus is recommended. You should also be familiar with how Neural Networks work. For reference, you can visit the [Python](https://docs.python.org/dev/tutorial/index.html), [Linear algebra on n-dimensional arrays](https://numpy.org/numpy-tutorials/content/tutorial-svd.html) and [Calculus](https://d2l.ai/chapter_appendix-mathematics-for-deep-learning/multivariable-calculus.html) tutorials. To get a refresher on Deep Learning basics, You should consider reading [the d2l.ai book](https://d2l.ai/chapter_recurrent-neural-networks/index.html), which is an interactive deep learning book with multi-framework code, math, and discussions. You can also go through the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) to understand how a basic neural network is implemented from scratch. @@ -101,8 +107,8 @@ We made sure to include different demographics in our data and included a range 1. **Text Denoising** : Before converting your text into vectors, it is important to clean it and remove all unhelpful parts a.k.a the noise from your data by converting all characters to lowercase, removing html tags, brackets and stop words (words that don't add much meaning to a sentence). Without this step the dataset is often a cluster of words that the computer doesn't understand. -2. **Converting words to vectors** : A word embedding is a learned representation for text where words that have the same meaning have a similar representation. Individual words are represented as real-valued vectors in a predefined vector space. GloVe is an unsupervised algorithm developed by Stanford for generating word embeddings by generating global word-word co-occurence matrix from a corpus. You can download the zipped files containing the embeddings from https://nlp.stanford.edu/projects/glove/. Here you can choose any of the four options for different sizes or training datasets. We have chosen the least memory consuming embedding file. - >The GloVe word embeddings include sets that were trained on billions of tokens, some up to 840 billion tokens. These algorithms exhibit stereotypical biases, such as gender bias which can be traced back to the original training data. For example certain occupations seem to be more biased towards a particular gender, reinforcing problematic stereotypes. The nearest solution to this problem are some de-biasing algorithms as the one presented in https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1184/reports/6835575.pdf which one can use on embeddings of their choice to mitigate bias, if present. +2. **Converting words to vectors** : A word embedding is a learned representation for text where words that have the same meaning have a similar representation. Individual words are represented as real-valued vectors in a predefined vector space. GloVe is an unsupervised algorithm developed by Stanford for generating word embeddings by generating global word-word co-occurence matrix from a corpus. You can download the zipped files containing the embeddings from [the GloVe official website](https://nlp.stanford.edu/projects/glove/). Here you can choose any of the four options for different sizes or training datasets. We have chosen the least memory consuming embedding file. + >The GloVe word embeddings include sets that were trained on billions of tokens, some up to 840 billion tokens. These algorithms exhibit stereotypical biases, such as gender bias which can be traced back to the original training data. For example certain occupations seem to be more biased towards a particular gender, reinforcing problematic stereotypes. The nearest solution to this problem are some de-biasing algorithms as the one presented in [this research article](https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1184/reports/6835575.pdf), which one can use on embeddings of their choice to mitigate bias, if present. You'll start with importing the necessary packages to build our Deep Learning network. @@ -435,7 +441,7 @@ emb_path = textproc.unzipper(glove, 'glove.6B.300d.txt') emb_matrix = textproc.loadGloveModel(emb_path) ``` -## 3. Build the Deep Learning Model¶ +## 3. Build the Deep Learning Model It is time to start implementing our LSTM! You will have to first familiarize yourself with some high-level concepts of the basic building blocks of a deep learning model. You can refer to the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) for the same. You will then learn how a Recurrent Neural Network differs from a plain Neural Network and what makes it so suitable for processing sequential data. Afterwards, you will construct the building blocks of a simple deep learning model in Python and NumPy and train it to learn to classify the sentiment of a piece of text as positive or negative with a certain level of accuracy @@ -1043,11 +1049,11 @@ To further enhance and optimize your neural network model, you can consider one - Initialize weights using [Xavier Initialization](https://d2l.ai/chapter_multilayer-perceptrons/numerical-stability-and-init.html#xavier-initialization) to prevent vanishing/exploding gradients instead of initializing them randomly. - Replace LSTM with a [Bidirectional LSTM](https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks) to use both left and right context for predicting sentiment. -Nowadays, LSTMs have been replaced by the [Transformer](https://jalammar.github.io/illustrated-transformer/)( which uses [Attention](https://jalammar.github.io/visualizing-neural-machine-translation-mechanics-of-seq2seq-models-with-attention/) to tackle all the problems that plague an LSTM such as as lack of [transfer learning](https://en.wikipedia.org/wiki/Transfer_learning), lack of [parallel training](https://web.stanford.edu/~rezab/classes/cme323/S16/projects_reports/hedge_usmani.pdf) and a long gradient chain for lengthy sequences +Nowadays, LSTMs have been replaced by the [Transformer](https://jalammar.github.io/illustrated-transformer/) which uses [Attention](https://jalammar.github.io/visualizing-neural-machine-translation-mechanics-of-seq2seq-models-with-attention/) to tackle all the problems that plague an LSTM such as lack of [transfer learning](https://en.wikipedia.org/wiki/Transfer_learning), lack of [parallel training](https://web.stanford.edu/~rezab/classes/cme323/S16/projects_reports/hedge_usmani.pdf), and a long gradient chain for lengthy sequences. -Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as PyTorch, JAX, TensorFlow or MXNet — that provide NumPy-like APIs, have built-in automatic differentiation and GPU support, and are designed for high-performance numerical computing and machine learning. +Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as PyTorch, JAX or TensorFlow — that provide NumPy-like APIs, have built-in automatic differentiation and GPU support, and are designed for high-performance numerical computing and machine learning. Finally, to know more about how ethics come into play when developing a machine learning model, you can refer to the following resources : -- Data ethics resources by the Turing Institute. https://www.turing.ac.uk/research/data-ethics +- [Data ethics resources](https://www.turing.ac.uk/research/data-ethics) by the Turing Institute - Considering how artificial intelligence shifts power, an [article](https://www.nature.com/articles/d41586-020-02003-2) and [talk](https://slideslive.com/38923453/the-values-of-machine-learning) by Pratyusha Kalluri - More ethics resources on [this blog post](https://www.fast.ai/2018/09/24/ai-ethics-resources/) by Rachel Thomas and the [Radical AI podcast](https://www.radicalai.org/) diff --git a/content/tutorial-plotting-fractals.md b/content/tutorial-plotting-fractals.md index e97b0cbe..a1921cea 100644 --- a/content/tutorial-plotting-fractals.md +++ b/content/tutorial-plotting-fractals.md @@ -301,14 +301,14 @@ For example, setting $c = \frac{\pi}{10}$ gives us a very elegant cloud shape, w ```{code-cell} ipython3 output = julia(mesh, c=np.pi/10, num_iter=20) -kwargs = {'title': 'f(z) = z^2 + \dfrac{\pi}{10}', 'cmap': 'plasma'} +kwargs = {'title': r'f(z) = z^2 + \dfrac{\pi}{10}', 'cmap': 'plasma'} plot_fractal(output, **kwargs); ``` ```{code-cell} ipython3 output = julia(mesh, c=-0.75 + 0.4j, num_iter=20) -kwargs = {'title': 'f(z) = z^2 - \dfrac{3}{4} + 0.4i', 'cmap': 'Greens_r'} +kwargs = {'title': r'f(z) = z^2 - \dfrac{3}{4} + 0.4i', 'cmap': 'Greens_r'} plot_fractal(output, **kwargs); ``` @@ -334,7 +334,7 @@ def mandelbrot(mesh, num_iter=10, radius=2): ```{code-cell} ipython3 output = mandelbrot(mesh, num_iter=50) -kwargs = {'title': 'Mandelbrot \ set', 'cmap': 'hot'} +kwargs = {'title': 'Mandelbrot \\ set', 'cmap': 'hot'} plot_fractal(output, **kwargs); ``` @@ -370,8 +370,6 @@ for deg, ax in enumerate(axes.ravel()): diverge_len = general_julia(mesh, f=power, num_iter=15) ax.imshow(diverge_len, extent=[-2, 2, -2, 2], cmap='binary') ax.set_title(f'$f(z) = z^{degree} -1$') - -fig.tight_layout(); ``` Needless to say, there is a large amount of exploring that can be done by fiddling with the inputted function, value of $c$, number of iterations, radius and even the density of the mesh and choice of colours. @@ -419,7 +417,7 @@ p.deriv() ```{code-cell} ipython3 output = newton_fractal(mesh, p, p.deriv(), num_iter=15, r=2) -kwargs = {'title': 'f(z) = z - \dfrac{(z^8 + 15z^4 - 16)}{(8z^7 + 60z^3)}', 'cmap': 'copper'} +kwargs = {'title': r'f(z) = z - \dfrac{(z^8 + 15z^4 - 16)}{(8z^7 + 60z^3)}', 'cmap': 'copper'} plot_fractal(output, **kwargs) ``` @@ -443,7 +441,7 @@ def d_tan(z): ```{code-cell} ipython3 output = newton_fractal(mesh, f_tan, d_tan, num_iter=15, r=50) -kwargs = {'title': 'f(z) = z - \dfrac{sin(z)cos(z)}{2}', 'cmap': 'binary'} +kwargs = {'title': r'f(z) = z - \dfrac{sin(z)cos(z)}{2}', 'cmap': 'binary'} plot_fractal(output, **kwargs); ``` @@ -475,7 +473,7 @@ We will denote this one 'Wacky fractal', as its equation would not be fun to try ```{code-cell} ipython3 output = newton_fractal(small_mesh, sin_sum, d_sin_sum, num_iter=10, r=1) -kwargs = {'title': 'Wacky \ fractal', 'figsize': (6, 6), 'extent': [-1, 1, -1, 1], 'cmap': 'terrain'} +kwargs = {'title': 'Wacky \\ fractal', 'figsize': (6, 6), 'extent': [-1, 1, -1, 1], 'cmap': 'terrain'} plot_fractal(output, **kwargs) ``` @@ -550,7 +548,7 @@ def accident(z): ```{code-cell} ipython3 output = general_julia(mesh, f=accident, num_iter=15, c=0, radius=np.pi) -kwargs = {'title': 'Accidental \ fractal', 'cmap': 'Blues'} +kwargs = {'title': 'Accidental \\ fractal', 'cmap': 'Blues'} plot_fractal(output, **kwargs); ``` diff --git a/content/tutorial-static_equilibrium.md b/content/tutorial-static_equilibrium.md index 0e8b82f7..f649e56e 100644 --- a/content/tutorial-static_equilibrium.md +++ b/content/tutorial-static_equilibrium.md @@ -263,7 +263,7 @@ print("Reaction moment =", M) ``` ### Another Example -Let's look at a slightly more complicated model. In this example you will be observing a beam with two cables and an applied force. This time you need to find both the tension in the cords and the reaction forces of the beam. *(Source: [Vector Mechanics for Engineers: Statics](https://www.mheducation.com/highered/product/vector-mechanics-engineers-statics-beer-johnston/M9780077687304.html), Problem 4.106)* +Let's look at a slightly more complicated model. In this example you will be observing a beam with two cables and an applied force. This time you need to find both the tension in the cords and the reaction forces of the beam. *(Source: [Vector Mechanics for Engineers: Statics and Dynamics](https://www.mheducation.com/highered/product/Vector-Mechanics-for-Engineers-Statics-and-Dynamics-Beer.html), Problem 4.106)* ![image.png](_static/problem4.png) @@ -387,5 +387,5 @@ This same process can be applied to kinetic problems or in any number of dimensi ### References -1. [Vector Mechanics for Engineers: Statics (Beer & Johnston & Mazurek)](https://www.mheducation.com/highered/product/vector-mechanics-engineers-statics-beer-johnston/M9780077687304.html) +1. [Vector Mechanics for Engineers: Statics and Dynamics (Beer & Johnston & Mazurek & et al.)](https://www.mheducation.com/highered/product/Vector-Mechanics-for-Engineers-Statics-and-Dynamics-Beer.html) 2. [NumPy Reference](https://numpy.org/doc/stable/reference/) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 3a0b58cd..aded8df0 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -35,15 +35,19 @@ After this tutorial, you should be able to: ## Content -In this tutorial, we will use a [matrix decomposition](https://en.wikipedia.org/wiki/Matrix_decomposition) from linear algebra, the Singular Value Decomposition, to generate a compressed approximation of an image. We'll use the `face` image from the [scipy.misc](https://docs.scipy.org/doc/scipy/reference/misc.html#module-scipy.misc) module: +In this tutorial, we will use a [matrix decomposition](https://en.wikipedia.org/wiki/Matrix_decomposition) from linear algebra, the Singular Value Decomposition, to generate a compressed approximation of an image. We'll use the `face` image from the [scipy.datasets](https://docs.scipy.org/doc/scipy/reference/datasets.html) module: ```{code-cell} -from scipy import misc +# TODO: Rm try-except with scipy 1.10 is the minimum supported version +try: + from scipy.datasets import face +except ImportError: # Data was in scipy.misc prior to scipy v1.10 + from scipy.misc import face -img = misc.face() +img = face() ``` -**Note**: If you prefer, you can use your own image as you work through this tutorial. In order to transform your image into a NumPy array that can be manipulated, you can use the `imread` function from the [matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. Alternatively, you can use the [imageio.imread](https://imageio.readthedocs.io/en/stable/userapi.html#imageio.imread) function from the `imageio` library. Be aware that if you use your own image, you'll likely need to adapt the steps below. For more information on how images are treated when converted to NumPy arrays, see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. +**Note**: If you prefer, you can use your own image as you work through this tutorial. In order to transform your image into a NumPy array that can be manipulated, you can use the `imread` function from the [matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. Alternatively, you can use the [imageio.imread](https://imageio.readthedocs.io/en/stable/_autosummary/imageio.v3.imread.html) function from the `imageio` library. Be aware that if you use your own image, you'll likely need to adapt the steps below. For more information on how images are treated when converted to NumPy arrays, see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. +++ @@ -91,7 +95,7 @@ img[:, :, 0] ``` From the output above, we can see that every value in `img[:, :, 0]` is an integer value between 0 and 255, representing the level of red in each corresponding image pixel (keep in mind that this might be different if you -use your own image instead of [scipy.misc.face](https://docs.scipy.org/doc/scipy/reference/generated/scipy.misc.face.html#scipy.misc.face)). +use your own image instead of [scipy.datasets.face](https://docs.scipy.org/doc/scipy/reference/generated/scipy.datasets.face.html)). As expected, this is a 768x1024 matrix: @@ -134,7 +138,7 @@ It is possible to use methods from linear algebra to approximate an existing set +++ -**Note**: We will use NumPy's linear algebra module, [numpy.linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg), to perform the operations in this tutorial. Most of the linear algebra functions in this module can also be found in [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg), and users are encouraged to use the [scipy](https://docs.scipy.org/doc/scipy/reference/index.html#module-scipy) module for real-world applications. However, some functions in the [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg) module, such as the SVD function, only support 2D arrays. For more information on this, check the [scipy.linalg Reference](https://docs.scipy.org/doc/scipy/reference/tutorial/linalg.html). +**Note**: We will use NumPy's linear algebra module, [numpy.linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg), to perform the operations in this tutorial. Most of the linear algebra functions in this module can also be found in [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg), and users are encouraged to use the [scipy](https://docs.scipy.org/doc/scipy/reference/index.html#module-scipy) module for real-world applications. However, some functions in the [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg) module, such as the SVD function, only support 2D arrays. For more information on this, check the [scipy.linalg page](https://docs.scipy.org/doc/scipy/tutorial/linalg.html). +++ @@ -150,7 +154,7 @@ $$U \Sigma V^T = A$$ where $U$ and $V^T$ are square and $\Sigma$ is the same size as $A$. $\Sigma$ is a diagonal matrix and contains the [singular values](https://en.wikipedia.org/wiki/Singular_value) of $A$, organized from largest to smallest. These values are always non-negative and can be used as an indicator of the "importance" of some features represented by the matrix $A$. -Let's see how this works in practice with just one matrix first. Note that according to [colorimetry](https://en.wikipedia.org/wiki/Grayscale#Colorimetric_(perceptual_luminance-reserving)_conversion_to_grayscale), +Let's see how this works in practice with just one matrix first. Note that according to [colorimetry](https://en.wikipedia.org/wiki/Grayscale#Colorimetric_(perceptual_luminance-preserving)_conversion_to_grayscale), it is possible to obtain a fairly reasonable grayscale version of our color image if we apply the formula $$Y = 0.2126 R + 0.7152 G + 0.0722 B$$ @@ -379,6 +383,6 @@ terms of the norm of the difference. For more information, see *G. H. Golub and - [Python tutorial](https://docs.python.org/dev/tutorial/index.html) - [NumPy Reference](https://numpy.org/devdocs/reference/index.html#reference) -- [SciPy Tutorial](https://docs.scipy.org/doc/scipy/reference/tutorial/index.html) +- [SciPy Tutorial](https://docs.scipy.org/doc/scipy/tutorial/index.html) - [SciPy Lecture Notes](https://scipy-lectures.org) - [A matlab, R, IDL, NumPy/SciPy dictionary](http://mathesaurus.sf.net/) diff --git a/content/tutorial-x-ray-image-processing.md b/content/tutorial-x-ray-image-processing.md index c8dd3a2b..9e00ea6c 100644 --- a/content/tutorial-x-ray-image-processing.md +++ b/content/tutorial-x-ray-image-processing.md @@ -187,7 +187,7 @@ notebook: ```{code-cell} ipython3 GIF_PATH = os.path.join(DIR, "xray_image.gif") -imageio.mimwrite(GIF_PATH, combined_xray_images_1, format= ".gif", fps=1) +imageio.mimwrite(GIF_PATH, combined_xray_images_1, format= ".gif", duration=1000) ``` Which gives us: diff --git a/content/x_y-squared.csv b/content/x_y-squared.csv deleted file mode 100644 index e74126ff..00000000 --- a/content/x_y-squared.csv +++ /dev/null @@ -1,11 +0,0 @@ -# x, y -0.000000000000000000e+00,0.000000000000000000e+00 -1.000000000000000000e+00,1.000000000000000000e+00 -2.000000000000000000e+00,4.000000000000000000e+00 -3.000000000000000000e+00,9.000000000000000000e+00 -4.000000000000000000e+00,1.600000000000000000e+01 -5.000000000000000000e+00,2.500000000000000000e+01 -6.000000000000000000e+00,3.600000000000000000e+01 -7.000000000000000000e+00,4.900000000000000000e+01 -8.000000000000000000e+00,6.400000000000000000e+01 -9.000000000000000000e+00,8.100000000000000000e+01 diff --git a/content/x_y-squared.npz b/content/x_y-squared.npz deleted file mode 100644 index 6c32f196..00000000 Binary files a/content/x_y-squared.npz and /dev/null differ diff --git a/environment.yml b/environment.yml index 1a8abe27..15dd1085 100644 --- a/environment.yml +++ b/environment.yml @@ -5,12 +5,14 @@ dependencies: # For running the tutorials - numpy - scipy + - pooch - matplotlib - - pandas - - statsmodels + - pandas - imageio # For building the site - - sphinx<5 + - sphinx - myst-nb - sphinx-book-theme - sphinx-copybutton + # to load the md files in binder + - jupytext diff --git a/ignore_testing b/ignore_testing new file mode 100644 index 00000000..a72bcd51 --- /dev/null +++ b/ignore_testing @@ -0,0 +1,4 @@ +content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md +content/pairing.md +content/tutorial-style-guide.md +content/tutorial-nlp-from-scratch.md diff --git a/requirements.txt b/requirements.txt index 63b2eefa..dd03cc89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ # For the tutorials numpy scipy +pooch # for scipy.datasets matplotlib pandas -statsmodels imageio # For supporting .md-based notebooks jupytext diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 00000000..55090899 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.10 diff --git a/site/applications.md b/site/applications.md index 743143b0..fa81374e 100644 --- a/site/applications.md +++ b/site/applications.md @@ -10,8 +10,6 @@ maxdepth: 1 content/mooreslaw-tutorial content/tutorial-deep-learning-on-mnist -content/tutorial-deep-reinforcement-learning-with-pong-from-pixels -content/tutorial-nlp-from-scratch content/tutorial-x-ray-image-processing content/tutorial-static_equilibrium content/tutorial-plotting-fractals diff --git a/site/articles.md b/site/articles.md new file mode 100644 index 00000000..8540ed69 --- /dev/null +++ b/site/articles.md @@ -0,0 +1,13 @@ +# Articles + +```{admonition} Help improve the tutorials! + +Want to make a valuable contribution to the tutorials? Consider working on +these articles so that they become fully executable/reproducible! +``` + +```{toctree} + +content/tutorial-deep-reinforcement-learning-with-pong-from-pixels +content/tutorial-nlp-from-scratch +``` diff --git a/site/conf.py b/site/conf.py index b9ab2a01..4aea25ca 100644 --- a/site/conf.py +++ b/site/conf.py @@ -12,9 +12,10 @@ # -- Project information ----------------------------------------------------- +from datetime import date project = 'NumPy tutorials' -copyright = '2020, the NumPy community' +copyright = f'2020-{date.today().year}, the NumPy community' author = 'the NumPy community' @@ -64,7 +65,7 @@ "repository_branch": "main", "use_repository_button": True, "use_issues_button": True, - "use_edit_page_button": True, + "use_edit_page_button": False, "path_to_docs": "site/", "launch_buttons": { "binderhub_url": "https://mybinder.org", diff --git a/site/contributing.md b/site/contributing.md index 8985c56c..e95774a5 100644 --- a/site/contributing.md +++ b/site/contributing.md @@ -29,7 +29,7 @@ used in the main NumPy documentation has two reasons: * Jupyter notebooks are a common format for communicating scientific information. - * Jupyter notebooks can be launched in [Binder](https://www.mybinder.org), so that users can interact + * Jupyter notebooks can be launched in [Binder](https://mybinder.org), so that users can interact with tutorials * rST may present a barrier for some people who might otherwise be very interested in contributing tutorial material. @@ -42,7 +42,7 @@ You may notice our content is in markdown format (`.md` files). We review and host notebooks in the [MyST-NB](https://myst-nb.readthedocs.io/) format. We accept both Jupyter notebooks (`.ipynb`) and MyST-NB notebooks (`.md`). If you want to sync your `.ipynb` to your `.md` file follow the [pairing -tutorial](content/pairing.md). +tutorial](content/pairing). ```{toctree} :hidden: diff --git a/site/index.md b/site/index.md index 6d6baa3a..c15d6877 100644 --- a/site/index.md +++ b/site/index.md @@ -31,6 +31,22 @@ applications contributing ``` +### Non-executable articles + +```{admonition} Help improve the tutorials! + +Want to make a valuable contribution to the tutorials? Consider contributing to +these existing articles to help make them fully executable and reproducible! +``` + +```{toctree} +--- +maxdepth: 2 +--- + +articles +``` + ## Useful links and resources The following links may be useful: diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..d20d5b24 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,2 @@ +pytest +nbval diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..ff48327d --- /dev/null +++ b/tox.ini @@ -0,0 +1,49 @@ +[tox] +envlist = + py{39,310,311,312}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} +requires = + pip >= 19.3.1 + +[testenv] + +description = run tests + +setenv = + devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple + +deps = + # We use these files to specify all the dependencies, and below we override + # versions for specific testing schenarios + -rtest_requirements.txt + -rsite/requirements.txt + -rrequirements.txt + + oldestdeps: numpy==1.23 + oldestdeps: matplotlib==3.6 + oldestdeps: scipy==1.8 + oldestdeps: pandas==1.4 + + devdeps: numpy>=0.0.dev0 + devdeps: scipy>=0.0.dev0 + devdeps: matplotlib>=0.0.dev0 + devdeps: pandas>=0.0.dev0 + +allowlist_externals = bash, make + +commands = + # Force numpy reinstall to work around upper version limits in downstream dependencies (e.g. pandas) + devdeps: pip install -U --pre --no-deps --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy + + pip freeze + + # Ignore testing the tutorials listed in ignore_testing file + !buildhtml: bash -c 'find content -name "*.md" | grep -vf ignore_testing | xargs jupytext --to notebook ' + + !buildhtml: pytest --nbval-lax --durations=10 content/ + buildhtml: make -C site/ SPHINXOPTS="-nWT --keep-going" html + +pip_pre = + predeps: true + !predeps: false + +skip_install = true