From 329cf41af752c8d58e92c6a1e28fc116f8eff4a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 7 Sep 2022 17:03:20 -0700 Subject: [PATCH 001/152] Adding basic test configuration to tox.ini --- ignore_testing | 4 ++++ tox.ini | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 ignore_testing create mode 100644 tox.ini diff --git a/ignore_testing b/ignore_testing new file mode 100644 index 00000000..a72bcd51 --- /dev/null +++ b/ignore_testing @@ -0,0 +1,4 @@ +content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md +content/pairing.md +content/tutorial-style-guide.md +content/tutorial-nlp-from-scratch.md diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..9cffae32 --- /dev/null +++ b/tox.ini @@ -0,0 +1,38 @@ +[tox] +envlist = + py{38,39,310}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} +requires = + pip >= 19.3.1 + +[testenv] + +description = run tests + +deps = + # We use these files to specify all the dependencies, and below we override + # versions for specific testing schenarios + -rsite/requirements.txt + -rrequirements.txt + + # TODO: be clever and use the nightly wheels instead + devdeps: git+https://github.com/numpy/numpy.git#egg=numpy + + # TODO: add the oldest supported versions of all the dependencies here + # oldestdeps: numpy==1.18 + # oldestdeps: matplotlib==3.1.2 + # oldestdeps: scipy==1.4 + +commands = + pip freeze + + # Ignore testing the tutorials listed in ignore_testing file + !buildhtml: bash -c 'find content -name "*.md" | grep -vf ignore_testing | xargs jupytext --to notebook ' + + !buildhtml: pytest --nbval-lax --durations=10 content/ + buildhtml: make -C site/ SPHINXOPTS="-nWT --keep-going" html + +pip_pre = + predeps: true + !predeps: false + +skip_install = true From 21168e5122c9cf66694919c005b9fc7a6931423c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 7 Sep 2022 17:18:10 -0700 Subject: [PATCH 002/152] Converting the current GHA test matrix to use tox --- .github/workflows/notebooks.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 9489bff9..b38678fa 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -18,6 +18,7 @@ jobs: matrix: os: [Ubuntu-20.04, macOS-latest] python-version: [3.8, 3.9, "3.10"] + toxenv: test steps: - uses: actions/checkout@v2 @@ -27,18 +28,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: | - python -m pip install -r site/requirements.txt -r requirements.txt - python -m pip list + run: python -m pip install --upgrade tox - name: Test with nbval - run: | - python -m pip install pytest nbval - find content/ -name "*.md" -exec jupytext --to notebook {} \; - # TODO: find better way to exclude notebooks from test - rm content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.ipynb - rm content/pairing.ipynb - rm content/tutorial-style-guide.ipynb - rm content/tutorial-nlp-from-scratch.ipynb - # Test notebook execution - pytest --nbval-lax --durations=10 content/ + run: tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }} From 2f524faaa03f72b04186263da5ac348b52d54790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 7 Sep 2022 17:39:36 -0700 Subject: [PATCH 003/152] Reworking the gha matrix --- .github/workflows/notebooks.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index b38678fa..86eb5eca 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -17,8 +17,15 @@ jobs: max-parallel: 12 matrix: os: [Ubuntu-20.04, macOS-latest] - python-version: [3.8, 3.9, "3.10"] - toxenv: test + include: + - python-version: '3.8' + toxenv: py38-test-oldestdeps + + - python-version: '3.9' + toxenv: py39-test + + - python-version: '3.10' + toxenv: py310-test steps: - uses: actions/checkout@v2 From 87cea397c380b83063e896f82118184c9c26abcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 7 Sep 2022 17:43:51 -0700 Subject: [PATCH 004/152] Specifying test requirements --- test_requirements.txt | 2 ++ tox.ini | 1 + 2 files changed, 3 insertions(+) create mode 100644 test_requirements.txt diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..d20d5b24 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,2 @@ +pytest +nbval diff --git a/tox.ini b/tox.ini index 9cffae32..a4e623d0 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ description = run tests deps = # We use these files to specify all the dependencies, and below we override # versions for specific testing schenarios + -rtest_requirements.txt -rsite/requirements.txt -rrequirements.txt From 8901a9197f98eae36b62c558d89d535c407ad4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 21 Sep 2022 08:12:03 -0700 Subject: [PATCH 005/152] CI: using nightly wheels for testing --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index a4e623d0..5d75fc75 100644 --- a/tox.ini +++ b/tox.ini @@ -15,8 +15,9 @@ deps = -rsite/requirements.txt -rrequirements.txt - # TODO: be clever and use the nightly wheels instead - devdeps: git+https://github.com/numpy/numpy.git#egg=numpy + devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy + devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple scipy + devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple matplotlib # TODO: add the oldest supported versions of all the dependencies here # oldestdeps: numpy==1.18 From 36b441e51c0f4cf5e65769f2d87700937ec9993e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 21 Sep 2022 08:14:14 -0700 Subject: [PATCH 006/152] MAINT: adding more generated content to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index fe8151b1..a1654131 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,6 @@ site/notebooks/* content/mooreslaw_regression* content/tutorial-x-ray-image-processing/xray_image.gif content/video +content/*ipynb +content/tutorial-nlp-from-scratch/parameters.npy +content/tutorial-nlp-from-scratch/*ipynb \ No newline at end of file From bb2b4b7039da7ed1911c6f3a0ee26b96af689f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 21 Sep 2022 12:06:15 -0700 Subject: [PATCH 007/152] CI: run dev version testing in CI --- .github/workflows/notebooks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 86eb5eca..9c8377ec 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -25,7 +25,7 @@ jobs: toxenv: py39-test - python-version: '3.10' - toxenv: py310-test + toxenv: py310-test-devdeps steps: - uses: actions/checkout@v2 From 466e6b4409891b5b3435d0c8aaf9d13cc023afea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 16 Nov 2022 18:32:41 -0800 Subject: [PATCH 008/152] CI: moving devdeps installs to commans --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 5d75fc75..71b8a547 100644 --- a/tox.ini +++ b/tox.ini @@ -15,16 +15,16 @@ deps = -rsite/requirements.txt -rrequirements.txt - devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple scipy - devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple matplotlib - # TODO: add the oldest supported versions of all the dependencies here # oldestdeps: numpy==1.18 # oldestdeps: matplotlib==3.1.2 # oldestdeps: scipy==1.4 commands = + devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy + devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple scipy + devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple matplotlib + pip freeze # Ignore testing the tutorials listed in ignore_testing file From e0b0d6aa38c43fdbbd1b972965c03068c8770bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 16 Nov 2022 19:42:09 -0800 Subject: [PATCH 009/152] CI: updating actions versions (#153) --- .github/workflows/conda.yml | 5 ++--- .github/workflows/notebooks.yml | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index c58fbc9a..55d773fe 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -21,7 +21,7 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true @@ -40,7 +40,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: sphinx-build-artifact path: site/_build/html/reports @@ -48,4 +48,3 @@ jobs: - name: fail on build errors if: steps.build_step.outcome != 'success' run: exit 1 - diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 9489bff9..337591db 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -20,9 +20,9 @@ jobs: python-version: [3.8, 3.9, "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 9341046cd08b30e125903cbf8e0d15ce29019611 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 17 Nov 2022 12:44:38 -0800 Subject: [PATCH 010/152] MAINT: Replace polyfit/polyval with Polynomial class. The Polynomial class is recommended over the poly* functional interface for new code. --- content/tutorial-ma.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/content/tutorial-ma.md b/content/tutorial-ma.md index f4aa9a98..28260673 100644 --- a/content/tutorial-ma.md +++ b/content/tutorial-ma.md @@ -268,10 +268,9 @@ Finally, we can use the [numpy.polyfit](https://numpy.org/devdocs/reference/gene ```{code-cell} t = np.arange(len(china_total)) -params = np.polyfit(t[~china_total.mask], valid, 3) -cubic_fit = np.polyval(params, t) +model = np.polynomial.Polynomial.fit(t[~china_total.mask], valid, 3) plt.plot(t, china_total) -plt.plot(t, cubic_fit, "--") +plt.plot(t, model(t), "--") ``` This plot is not so readable since the lines seem to be over each other, so let's summarize in a more elaborate plot. We'll plot the real data when @@ -279,10 +278,10 @@ available, and show the cubic fit for unavailable data, using this fit to comput ```{code-cell} plt.plot(t, china_total) -plt.plot(t[china_total.mask], cubic_fit[china_total.mask], "--", color="orange") -plt.plot(7, np.polyval(params, 7), "r*") +plt.plot(t[china_total.mask], model(t)[china_total.mask], "--", color="orange") +plt.plot(7, model(7), "r*") plt.xticks([0, 7, 13], dates[[0, 7, 13]]) -plt.yticks([0, np.polyval(params, 7), 10000, 17500]) +plt.yticks([0, model(7), 10000, 17500]) plt.legend(["Mainland China", "Cubic estimate", "7 days after start"]) plt.title( "COVID-19 cumulative cases from Jan 21 to Feb 3 2020 - Mainland China\n" From 49430a3df35d75d34c01d27b9c94a89e8fbd679a Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 17 Nov 2022 13:13:54 -0800 Subject: [PATCH 011/152] MAINT: Update text to match example. --- content/tutorial-ma.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/content/tutorial-ma.md b/content/tutorial-ma.md index 28260673..90a7fa31 100644 --- a/content/tutorial-ma.md +++ b/content/tutorial-ma.md @@ -264,11 +264,13 @@ Now, if we want to create a very simple approximation for this data, we should t dates[~china_total.mask] ``` -Finally, we can use the [numpy.polyfit](https://numpy.org/devdocs/reference/generated/numpy.polyfit.html#numpy.polyfit) and [numpy.polyval](https://numpy.org/devdocs/reference/generated/numpy.polyval.html#numpy.polyval) functions to create a cubic polynomial that fits the data as best as possible: +Finally, we can use the +[fitting functionality of the numpy.polynomial](https://numpy.org/doc/stable/reference/generated/numpy.polynomial.polynomial.Polynomial.fit.html) +package to create a cubic polynomial model that fits the data as best as possible: ```{code-cell} t = np.arange(len(china_total)) -model = np.polynomial.Polynomial.fit(t[~china_total.mask], valid, 3) +model = np.polynomial.Polynomial.fit(t[~china_total.mask], valid, deg=3) plt.plot(t, china_total) plt.plot(t, model(t), "--") ``` From 5c4223a77279be855fe10b83ec38c5c803306709 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 5 Dec 2022 22:46:37 -0800 Subject: [PATCH 012/152] BLD: temporary pin to fix lexer. --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 63b2eefa..0f958b62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ statsmodels imageio # For supporting .md-based notebooks jupytext +# Temporary fix for lexer errors +ipython!=8.7.0 From 00ad5045069d906b792bfa3f95512732cc245c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 12 Dec 2022 19:33:32 -0800 Subject: [PATCH 013/152] MAINT: fixing compatibility of tox v4 (#161) --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 71b8a547..a3c005b6 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,8 @@ deps = # oldestdeps: matplotlib==3.1.2 # oldestdeps: scipy==1.4 +allowlist_externals = bash + commands = devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple scipy From a692a1559cabe6e07e9a82d8f4db36404e3c2b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 12 Dec 2022 23:44:14 -0800 Subject: [PATCH 014/152] MAINT: adding python version for binder --- runtime.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 runtime.txt diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 00000000..55090899 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.10 From f1ea3e3e0b319a20e253b2d6e4243a8932b1186a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 5 Jan 2023 14:47:42 -0800 Subject: [PATCH 015/152] MAINT: executable books are not yet sphinx5 compatible, help version resolution here --- site/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/requirements.txt b/site/requirements.txt index e040deba..8514cae9 100644 --- a/site/requirements.txt +++ b/site/requirements.txt @@ -1,4 +1,4 @@ -sphinx +sphinx<5 myst-nb sphinx-book-theme sphinx-copybutton From 699525a9c9ee991e2a397712aa885270eab136bf Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 5 Jan 2023 15:39:04 -0800 Subject: [PATCH 016/152] STY: Idiomatic use of arange. 0 and 1 are the default start and step respectively, so no need to explicitly include them. --- content/save-load-arrays.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/save-load-arrays.md b/content/save-load-arrays.md index 1960d0de..434c7370 100644 --- a/content/save-load-arrays.md +++ b/content/save-load-arrays.md @@ -68,7 +68,7 @@ will assign `x` to the integers from 0 to 9 using [`np.arange`](https://numpy.org/doc/stable/reference/generated/numpy.arange.html). ```{code-cell} -x = np.arange(0, 10, 1) +x = np.arange(10) y = x ** 2 print(x) print(y) From 01aad12650d084864b87c81df782260bf4240392 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 5 Jan 2023 16:36:29 -0800 Subject: [PATCH 017/152] CI: Update circleci deploy key (#164) See: https://circleci.com/blog/january-4-2023-security-alert/ --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a8df5dab..9e5be957 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,7 @@ jobs: - add_ssh_keys: fingerprints: - db:84:df:44:ad:77:d0:aa:2d:81:c9:73:30:9d:21:37 + 5c:54:62:37:75:7f:4d:14:f4:07:82:1c:50:0d:ee:9b - run: name: deploy to gh-pages From a6df091245c3f8e5fe6c07c90c8b7691f1d17865 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sun, 15 Jan 2023 22:05:14 -0800 Subject: [PATCH 018/152] MAINT: move non-executable content to its own category. (#148) * MAINT: move non-executable content to its own category. * Add warnings to untested articles. --- ...inforcement-learning-with-pong-from-pixels.md | 8 ++++++++ content/tutorial-nlp-from-scratch.md | 6 ++++++ site/applications.md | 2 -- site/articles.md | 13 +++++++++++++ site/index.md | 16 ++++++++++++++++ 5 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 site/articles.md diff --git a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md index 3ef3de03..69607388 100644 --- a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md +++ b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md @@ -14,6 +14,14 @@ kernelspec: # Deep reinforcement learning with Pong from pixels +```{caution} + +This article is not currently tested due to licensing/installation issues with +the underlying `gym` and `atari-py` dependencies. +Help improve this article by developing an example with reduced dependency +footprint! +``` + This tutorial demonstrates how to implement a deep reinforcement learning (RL) agent from scratch using a policy gradient method that learns to play the [Pong](https://gym.openai.com/envs/Pong-v0/) video game using screen pixels as inputs with NumPy. Your Pong agent will obtain experience on the go using an [artificial neural network](https://en.wikipedia.org/wiki/Artificial_neural_network) as its [policy](https://en.wikipedia.org/wiki/Reinforcement_learning). Pong is a 2D game from 1972 where two players use "rackets" to play a form of table tennis. Each player moves the racket up and down the screen and tries to hit a ball in their opponent's direction by touching it. The goal is to hit the ball such that it goes past the opponent's racket (they miss their shot). According to the rules, if a player reaches 21 points, they win. In Pong, the RL agent that learns to play against an opponent is displayed on the right. diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index 865fd1c9..68a31d27 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -15,6 +15,12 @@ jupyter: # Sentiment Analysis on notable speeches of the last decade +```{caution} + +This article is not currently tested. Help improve this tutorial by making it +fully executable! +``` + This tutorial demonstrates how to build a simple Long Short Term memory network (LSTM) from scratch in NumPy to perform sentiment analysis on a socially relevant and ethically acquired dataset. Your deep learning model (the LSTM) is a form of a Recurrent Neural Network and will learn to classify a piece of text as positive or negative from the IMDB reviews dataset. The dataset contains 50,000 movie reviews and corresponding labels. Based on the numeric representations of these reviews and their corresponding labels (supervised learning) the neural network will be trained to learn the sentiment using forward propagation and backpropagation through time since we are dealing with sequential data here. The output will be a vector containing the probabilities that the text samples are positive. diff --git a/site/applications.md b/site/applications.md index 743143b0..fa81374e 100644 --- a/site/applications.md +++ b/site/applications.md @@ -10,8 +10,6 @@ maxdepth: 1 content/mooreslaw-tutorial content/tutorial-deep-learning-on-mnist -content/tutorial-deep-reinforcement-learning-with-pong-from-pixels -content/tutorial-nlp-from-scratch content/tutorial-x-ray-image-processing content/tutorial-static_equilibrium content/tutorial-plotting-fractals diff --git a/site/articles.md b/site/articles.md new file mode 100644 index 00000000..8540ed69 --- /dev/null +++ b/site/articles.md @@ -0,0 +1,13 @@ +# Articles + +```{admonition} Help improve the tutorials! + +Want to make a valuable contribution to the tutorials? Consider working on +these articles so that they become fully executable/reproducible! +``` + +```{toctree} + +content/tutorial-deep-reinforcement-learning-with-pong-from-pixels +content/tutorial-nlp-from-scratch +``` diff --git a/site/index.md b/site/index.md index 6d6baa3a..c15d6877 100644 --- a/site/index.md +++ b/site/index.md @@ -31,6 +31,22 @@ applications contributing ``` +### Non-executable articles + +```{admonition} Help improve the tutorials! + +Want to make a valuable contribution to the tutorials? Consider contributing to +these existing articles to help make them fully executable and reproducible! +``` + +```{toctree} +--- +maxdepth: 2 +--- + +articles +``` + ## Useful links and resources The following links may be useful: From 39c8d5da8478c07895bb9c09f0d256c6f259e602 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sun, 15 Jan 2023 22:10:26 -0800 Subject: [PATCH 019/152] Update circleci to Python 3.10 images. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9e5be957..716a2610 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: build-docs: working_directory: ~/repo docker: - - image: cimg/python:3.8 + - image: cimg/python:3.10 steps: - checkout @@ -49,7 +49,7 @@ jobs: deploy-docs: working_directory: ~/repo docker: - - image: circleci/python:3.8.5-buster + - image: cimg/python:3.10 steps: - checkout From 60295b24efe5fe016f18dae36bc51c303d25c4c6 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sun, 15 Jan 2023 22:12:13 -0800 Subject: [PATCH 020/152] Ubuntu-20.04 -> Ubuntu-latest. --- .github/workflows/notebooks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index 8fe7c5ac..6fe30072 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -16,7 +16,7 @@ jobs: strategy: max-parallel: 12 matrix: - os: [Ubuntu-20.04, macOS-latest] + os: [Ubuntu-latest, macOS-latest] include: - python-version: '3.8' toxenv: py38-test-oldestdeps From 82c5bee230cbe2c9512465573ec7c9bf0f032ee4 Mon Sep 17 00:00:00 2001 From: Oriol Abril-Pla Date: Mon, 16 Jan 2023 07:14:27 +0100 Subject: [PATCH 021/152] Fix link in svd tutorial (#167) * Fix link in svd tutorial Co-authored-by: Ross Barnowski --- content/tutorial-svd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 3a0b58cd..cafc9697 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -134,7 +134,7 @@ It is possible to use methods from linear algebra to approximate an existing set +++ -**Note**: We will use NumPy's linear algebra module, [numpy.linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg), to perform the operations in this tutorial. Most of the linear algebra functions in this module can also be found in [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg), and users are encouraged to use the [scipy](https://docs.scipy.org/doc/scipy/reference/index.html#module-scipy) module for real-world applications. However, some functions in the [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg) module, such as the SVD function, only support 2D arrays. For more information on this, check the [scipy.linalg Reference](https://docs.scipy.org/doc/scipy/reference/tutorial/linalg.html). +**Note**: We will use NumPy's linear algebra module, [numpy.linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg), to perform the operations in this tutorial. Most of the linear algebra functions in this module can also be found in [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg), and users are encouraged to use the [scipy](https://docs.scipy.org/doc/scipy/reference/index.html#module-scipy) module for real-world applications. However, some functions in the [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg) module, such as the SVD function, only support 2D arrays. For more information on this, check the [scipy.linalg page](https://docs.scipy.org/doc/scipy/tutorial/linalg.html). +++ From 707d7a33d21c236e9dbc67229a8ae069cf53a5ce Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sun, 15 Jan 2023 22:28:19 -0800 Subject: [PATCH 022/152] Rm temp IPython pin. --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0f958b62..63b2eefa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,3 @@ statsmodels imageio # For supporting .md-based notebooks jupytext -# Temporary fix for lexer errors -ipython!=8.7.0 From 81ec6ef204678a7f094af50b36075bb94d2d19ab Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sun, 15 Jan 2023 22:35:40 -0800 Subject: [PATCH 023/152] Bump copyright and license dates. --- LICENSE.txt | 2 +- site/conf.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 8ce64521..014d51c9 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2005-2020, NumPy Developers. +Copyright (c) 2005-2023, NumPy Developers. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/site/conf.py b/site/conf.py index b9ab2a01..e1fe5b63 100644 --- a/site/conf.py +++ b/site/conf.py @@ -12,9 +12,10 @@ # -- Project information ----------------------------------------------------- +from datetime import date project = 'NumPy tutorials' -copyright = '2020, the NumPy community' +copyright = f'2020-{date.today().year}, the NumPy community' author = 'the NumPy community' From 7c649030d01bdeba9a4f803308a958786f59a110 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sun, 15 Jan 2023 23:28:46 -0800 Subject: [PATCH 024/152] Use f-strings instead of str concat for train/eval summary. --- content/tutorial-deep-learning-on-mnist.md | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/content/tutorial-deep-learning-on-mnist.md b/content/tutorial-deep-learning-on-mnist.md index 82aea978..63dbdbce 100644 --- a/content/tutorial-deep-learning-on-mnist.md +++ b/content/tutorial-deep-learning-on-mnist.md @@ -544,17 +544,13 @@ for j in range(epochs): # Summarize error and accuracy metrics at each epoch print( - "\n" - + "Epoch: " - + str(j) - + " Training set error:" - + str(training_loss / float(len(training_images)))[0:5] - + " Training set accuracy:" - + str(training_accurate_predictions / float(len(training_images))) - + " Test set error:" - + str(test_loss / float(len(test_images)))[0:5] - + " Test set accuracy:" - + str(test_accurate_predictions / float(len(test_images))) + ( + f"Epoch: {j}\n" + f" Training set error: {training_loss / len(training_images):.3f}\n" + f" Training set accuracy: {training_accurate_predictions / len(training_images)}\n" + f" Test set error: {test_loss / len(test_images):.3f}\n" + f" Test set accuracy: {test_accurate_predictions / len(test_images)}" + ) ) ``` From 8d6e6d734e2076b5211ecafc8b7a05f8706cd704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 16 Mar 2023 14:58:22 -0700 Subject: [PATCH 025/152] MAINT: rename workflow to reflect purpose --- .github/workflows/{notebooks.yml => ci_tests_run_notebooks.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{notebooks.yml => ci_tests_run_notebooks.yml} (100%) diff --git a/.github/workflows/notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml similarity index 100% rename from .github/workflows/notebooks.yml rename to .github/workflows/ci_tests_run_notebooks.yml From 81753e38b83bc8048fe10e25460d435cbad36d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 16 Mar 2023 14:50:56 -0700 Subject: [PATCH 026/152] CI: fix matrix logic to run all the jobs --- .github/workflows/ci_tests_run_notebooks.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 6fe30072..f239e472 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -9,23 +9,30 @@ on: - main schedule: - cron: '0 5 * * 1' + workflow_dispatch: jobs: - build: + tests: + name: ${{ matrix.os }} ${{ matrix.name }} runs-on: ${{ matrix.os }} strategy: - max-parallel: 12 matrix: - os: [Ubuntu-latest, macOS-latest] + # Run all supported OS for one Python version, then add a few extra scenarios + os: [ubuntu-latest, macos-latest] + python-version: ['3.9'] + toxenv: [py39-test] + name: ['with Python 3.9',] + include: - python-version: '3.8' toxenv: py38-test-oldestdeps - - - python-version: '3.9' - toxenv: py39-test + name: with Python 3.8 and oldest versioned dependencies + os: ubuntu-latest - python-version: '3.10' toxenv: py310-test-devdeps + name: with Python 3.10 with developer versioned dependencies + os: ubuntu-latest steps: - uses: actions/checkout@v3 From d8cf97d078ea5f76791587d7ebbfee6aed2dcbed Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 20 Apr 2023 10:53:19 -0700 Subject: [PATCH 027/152] CI: Add CIRCLE_TOKEN to fix artifact redirection. This will only work if CIRCLE_TOKEN is available. --- .github/workflows/circleci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index 5115024b..be39709f 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -8,5 +8,6 @@ jobs: uses: larsoner/circleci-artifacts-redirector-action@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} + api-token: ${{ secrets.CIRCLE_TOKEN }} artifact-path: 0/site/_build/html/index.html circleci-jobs: build-docs From b51cb68586d0c207a9d7b5428d815b6af0bdb7a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 15 Mar 2023 12:33:32 -0700 Subject: [PATCH 028/152] CI: adding windows testing to CI --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index f239e472..f557bea0 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: # Run all supported OS for one Python version, then add a few extra scenarios - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] python-version: ['3.9'] toxenv: [py39-test] name: ['with Python 3.9',] From 188a39ea2a8d2eee44d5bcfa773eb8f4a3f4b426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 15 Mar 2023 12:29:06 -0700 Subject: [PATCH 029/152] CI: adding python 3.11 testing to CI --- .github/workflows/ci_tests_run_notebooks.yml | 9 +++++++-- tox.ini | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index f557bea0..8695b72d 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -30,8 +30,13 @@ jobs: os: ubuntu-latest - python-version: '3.10' - toxenv: py310-test-devdeps - name: with Python 3.10 with developer versioned dependencies + toxenv: py310-test + name: with Python 3.10 + os: ubuntu-latest + + - python-version: '3.11' + toxenv: py311-test-devdeps + name: with Python 3.11 and developer versioned dependencies os: ubuntu-latest steps: diff --git a/tox.ini b/tox.ini index a3c005b6..d1035f67 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{38,39,310}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} + py{38,39,310,311}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} requires = pip >= 19.3.1 From f0a34c9d2d6a60a581257a88469f2502e9b5fafc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 19 Apr 2023 18:31:09 -0700 Subject: [PATCH 030/152] CI: adding oldest dependencies check to matrix, and more devdeps dependencies, too --- tox.ini | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index d1035f67..769f71e7 100644 --- a/tox.ini +++ b/tox.ini @@ -15,10 +15,11 @@ deps = -rsite/requirements.txt -rrequirements.txt - # TODO: add the oldest supported versions of all the dependencies here - # oldestdeps: numpy==1.18 - # oldestdeps: matplotlib==3.1.2 - # oldestdeps: scipy==1.4 + oldestdeps: numpy==1.20 + oldestdeps: matplotlib==3.4 + oldestdeps: scipy==1.6 + oldestdeps: pandas==1.2 + oldestdeps: statsmodels==0.13 allowlist_externals = bash @@ -26,6 +27,8 @@ commands = devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple scipy devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple matplotlib + devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple pandas + devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple statsmodels pip freeze From 86fc4aaa3be05e425f6bbf90ab2482a5d1e115fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 20 Apr 2023 09:24:30 -0700 Subject: [PATCH 031/152] CI: dropping python 3.8 testing --- .github/workflows/ci_tests_run_notebooks.yml | 17 ++++++----------- tox.ini | 2 +- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 8695b72d..5051dc9e 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -19,19 +19,14 @@ jobs: matrix: # Run all supported OS for one Python version, then add a few extra scenarios os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.9'] - toxenv: [py39-test] - name: ['with Python 3.9',] + python-version: ['3.10'] + toxenv: [py310-test] + name: ['with Python 3.10',] include: - - python-version: '3.8' - toxenv: py38-test-oldestdeps - name: with Python 3.8 and oldest versioned dependencies - os: ubuntu-latest - - - python-version: '3.10' - toxenv: py310-test - name: with Python 3.10 + - python-version: '3.9' + toxenv: py39-test-oldestdeps + name: with Python 3.9 and oldest versioned dependencies os: ubuntu-latest - python-version: '3.11' diff --git a/tox.ini b/tox.ini index 769f71e7..b14c817d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{38,39,310,311}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} + py{39,310,311}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} requires = pip >= 19.3.1 From ab2acfdb844f455b7f9542b4ad9add776e0f0a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 24 Apr 2023 18:48:35 -0700 Subject: [PATCH 032/152] CI: let the full CI run even when a job is failing --- .github/workflows/ci_tests_run_notebooks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 5051dc9e..9b55c58f 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -16,6 +16,7 @@ jobs: name: ${{ matrix.os }} ${{ matrix.name }} runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: # Run all supported OS for one Python version, then add a few extra scenarios os: [ubuntu-latest, macos-latest, windows-latest] From 2bc7a2f4ea2f97d2efe8078e8c0b7dc79788c702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 24 Apr 2023 18:52:48 -0700 Subject: [PATCH 033/152] MAINT: temporarily limit imageIO version --- environment.yml | 5 +++-- requirements.txt | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 1a8abe27..00870fd6 100644 --- a/environment.yml +++ b/environment.yml @@ -6,9 +6,10 @@ dependencies: - numpy - scipy - matplotlib - - pandas + - pandas - statsmodels - - imageio + # Temporary version limit, see https://github.com/numpy/numpy-tutorials/issues/179 + - imageio<2.28 # For building the site - sphinx<5 - myst-nb diff --git a/requirements.txt b/requirements.txt index 63b2eefa..3eaadb83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ scipy matplotlib pandas statsmodels -imageio +# temporary version limit, see https://github.com/numpy/numpy-tutorials/issues/179 +imageio<2.28 # For supporting .md-based notebooks jupytext From 1c8b1c79dfaf41f791377302aced72529ce00f3d Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sat, 29 Apr 2023 20:27:11 -0700 Subject: [PATCH 034/152] MAINT: rm imageio pin and switch to durations kwarg. --- content/tutorial-x-ray-image-processing.md | 2 +- environment.yml | 3 +-- requirements.txt | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/content/tutorial-x-ray-image-processing.md b/content/tutorial-x-ray-image-processing.md index c8dd3a2b..9e00ea6c 100644 --- a/content/tutorial-x-ray-image-processing.md +++ b/content/tutorial-x-ray-image-processing.md @@ -187,7 +187,7 @@ notebook: ```{code-cell} ipython3 GIF_PATH = os.path.join(DIR, "xray_image.gif") -imageio.mimwrite(GIF_PATH, combined_xray_images_1, format= ".gif", fps=1) +imageio.mimwrite(GIF_PATH, combined_xray_images_1, format= ".gif", duration=1000) ``` Which gives us: diff --git a/environment.yml b/environment.yml index 00870fd6..90c83d4d 100644 --- a/environment.yml +++ b/environment.yml @@ -8,8 +8,7 @@ dependencies: - matplotlib - pandas - statsmodels - # Temporary version limit, see https://github.com/numpy/numpy-tutorials/issues/179 - - imageio<2.28 + - imageio # For building the site - sphinx<5 - myst-nb diff --git a/requirements.txt b/requirements.txt index 3eaadb83..63b2eefa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ scipy matplotlib pandas statsmodels -# temporary version limit, see https://github.com/numpy/numpy-tutorials/issues/179 -imageio<2.28 +imageio # For supporting .md-based notebooks jupytext From 3f173984eb34e654f3cdca814dbffaa4df52a047 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sun, 30 Apr 2023 09:54:56 -0700 Subject: [PATCH 035/152] MAINT: Unpin sphinx. --- environment.yml | 2 +- site/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 90c83d4d..8421695f 100644 --- a/environment.yml +++ b/environment.yml @@ -10,7 +10,7 @@ dependencies: - statsmodels - imageio # For building the site - - sphinx<5 + - sphinx - myst-nb - sphinx-book-theme - sphinx-copybutton diff --git a/site/requirements.txt b/site/requirements.txt index 8514cae9..e040deba 100644 --- a/site/requirements.txt +++ b/site/requirements.txt @@ -1,4 +1,4 @@ -sphinx<5 +sphinx myst-nb sphinx-book-theme sphinx-copybutton From 88780ee22698504e611142d2a2df1b1a66fcd432 Mon Sep 17 00:00:00 2001 From: partev Date: Fri, 26 May 2023 15:44:40 -0400 Subject: [PATCH 036/152] DOC: fix broken link to SciPy Tutorial fix broken link to SciPy Tutorial --- content/tutorial-svd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index cafc9697..a1fe60a4 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -379,6 +379,6 @@ terms of the norm of the difference. For more information, see *G. H. Golub and - [Python tutorial](https://docs.python.org/dev/tutorial/index.html) - [NumPy Reference](https://numpy.org/devdocs/reference/index.html#reference) -- [SciPy Tutorial](https://docs.scipy.org/doc/scipy/reference/tutorial/index.html) +- [SciPy Tutorial](https://docs.scipy.org/doc/scipy/tutorial/index.html) - [SciPy Lecture Notes](https://scipy-lectures.org) - [A matlab, R, IDL, NumPy/SciPy dictionary](http://mathesaurus.sf.net/) From 2285ba0725eaa8e9e0964850a10931a85f9153e2 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sun, 30 Jul 2023 19:23:49 -0700 Subject: [PATCH 037/152] Fix bug and improve computation / display of metrics for MNIST tutorial (#189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * BUG: Fix incorrect variable in computing eval metrics. * ENH: Replace list comps with vectorization. * ENH: Use dicts and condense plotting. * Update content/tutorial-deep-learning-on-mnist.md Co-authored-by: Brigitta Sipőcz --- content/tutorial-deep-learning-on-mnist.md | 46 +++++++++------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/content/tutorial-deep-learning-on-mnist.md b/content/tutorial-deep-learning-on-mnist.md index 63dbdbce..a41438c5 100644 --- a/content/tutorial-deep-learning-on-mnist.md +++ b/content/tutorial-deep-learning-on-mnist.md @@ -561,39 +561,31 @@ The training process may take many minutes, depending on a number of factors, su After executing the cell above, you can visualize the training and test set errors and accuracy for an instance of this training process. ```{code-cell} +epoch_range = np.arange(epochs) + 1 # Starting from 1 + # The training set metrics. -y_training_error = [ - store_training_loss[i] / float(len(training_images)) - for i in range(len(store_training_loss)) -] -x_training_error = range(1, len(store_training_loss) + 1) -y_training_accuracy = [ - store_training_accurate_pred[i] / float(len(training_images)) - for i in range(len(store_training_accurate_pred)) -] -x_training_accuracy = range(1, len(store_training_accurate_pred) + 1) +training_metrics = { + "accuracy": np.asarray(store_training_accurate_pred) / len(training_images), + "error": np.asarray(store_training_loss) / len(training_images), +} # The test set metrics. -y_test_error = [ - store_test_loss[i] / float(len(test_images)) for i in range(len(store_test_loss)) -] -x_test_error = range(1, len(store_test_loss) + 1) -y_test_accuracy = [ - store_training_accurate_pred[i] / float(len(training_images)) - for i in range(len(store_training_accurate_pred)) -] -x_test_accuracy = range(1, len(store_test_accurate_pred) + 1) +test_metrics = { + "accuracy": np.asarray(store_test_accurate_pred) / len(test_images), + "error": np.asarray(store_test_loss) / len(test_images), +} # Display the plots. fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(15, 5)) -axes[0].set_title("Training set error, accuracy") -axes[0].plot(x_training_accuracy, y_training_accuracy, label="Training set accuracy") -axes[0].plot(x_training_error, y_training_error, label="Training set error") -axes[0].set_xlabel("Epochs") -axes[1].set_title("Test set error, accuracy") -axes[1].plot(x_test_accuracy, y_test_accuracy, label="Test set accuracy") -axes[1].plot(x_test_error, y_test_error, label="Test set error") -axes[1].set_xlabel("Epochs") +for ax, metrics, title in zip( + axes, (training_metrics, test_metrics), ("Training set", "Test set") +): + # Plot the metrics + for metric, values in metrics.items(): + ax.plot(epoch_range, values, label=metric.capitalize()) + ax.set_title(title) + ax.set_xlabel("Epochs") + ax.legend() plt.show() ``` From 911d71aa04a938cbf38441bd802477685370189e Mon Sep 17 00:00:00 2001 From: D McMurchy <5430744+Zirescu@users.noreply.github.com> Date: Mon, 4 Sep 2023 19:46:10 -0700 Subject: [PATCH 038/152] Update contributing.md Removes the www on the URL for Binder as their website uses HSTS and their SSL Cert is only valid for mybinder.org and not www.mybinder.org. A user will not be able to directly access their website without editing the URL. --- site/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/contributing.md b/site/contributing.md index 8985c56c..c35db184 100644 --- a/site/contributing.md +++ b/site/contributing.md @@ -29,7 +29,7 @@ used in the main NumPy documentation has two reasons: * Jupyter notebooks are a common format for communicating scientific information. - * Jupyter notebooks can be launched in [Binder](https://www.mybinder.org), so that users can interact + * Jupyter notebooks can be launched in [Binder](https://mybinder.org), so that users can interact with tutorials * rST may present a barrier for some people who might otherwise be very interested in contributing tutorial material. From 4ea40a0a13d734a58276e795c2af5839e75c22d2 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 21 Dec 2023 12:08:39 -0800 Subject: [PATCH 039/152] ENH: Use numpy.polynomial for linear fit instead of statsmodels (#182) * Initial removal of statsmodels from mooreslaw md * Update wording for polynomial.fit. * Update wording, rm std. err. * MAINT: rm statsmodels from requirements/testing. --- content/mooreslaw-tutorial.md | 77 ++++++++--------------------------- environment.yml | 1 - requirements.txt | 1 - tox.ini | 2 - 4 files changed, 18 insertions(+), 63 deletions(-) diff --git a/content/mooreslaw-tutorial.md b/content/mooreslaw-tutorial.md index 18a13dcc..b9fefc8d 100644 --- a/content/mooreslaw-tutorial.md +++ b/content/mooreslaw-tutorial.md @@ -44,19 +44,17 @@ the 53 years following his prediction. You will determine the best-fit constants * NumPy * [Matplotlib](https://matplotlib.org/) -* [statsmodels](https://www.statsmodels.org) ordinary linear regression imported with the following commands ```{code-cell} import matplotlib.pyplot as plt import numpy as np -import statsmodels.api as sm ``` **2.** Since this is an exponential growth law you need a little background in doing math with [natural logs](https://en.wikipedia.org/wiki/Natural_logarithm) and [exponentials](https://en.wikipedia.org/wiki/Exponential_function). -You'll use these NumPy, Matplotlib, and statsmodels functions: +You'll use these NumPy and Matplotlib functions: * [`np.loadtxt`](https://numpy.org/doc/stable/reference/generated/numpy.loadtxt.html): this function loads text into a NumPy array * [`np.log`](https://numpy.org/doc/stable/reference/generated/numpy.log.html): this function takes the natural log of all elements in a NumPy array @@ -64,7 +62,6 @@ You'll use these NumPy, Matplotlib, and statsmodels functions: * [`lambda`](https://docs.python.org/3/library/ast.html?highlight=lambda#ast.Lambda): this is a minimal function definition for creating a function model * [`plt.semilogy`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.semilogy.html): this function will plot x-y data onto a figure with a linear x-axis and $\log_{10}$ y-axis [`plt.plot`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.plot.html): this function will plot x-y data on linear axes -* [`sm.OLS`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html): find fitting parameters and standard errors using the statsmodels ordinary least squares model * slicing arrays: view parts of the data loaded into the workspace, slice the arrays e.g. `x[:10]` for the first 10 values in the array, `x` * boolean array indexing: to view parts of the data that match a given condition use boolean operations to index an array * [`np.block`](https://numpy.org/doc/stable/reference/generated/numpy.block.html): to combine arrays into 2D arrays @@ -215,59 +212,31 @@ where $\mathbf{y}$ are the observations of the log of the number of transistors in a 1D array and $\mathbf{Z}=[\text{year}_i^1,~\text{year}_i^0]$ are the polynomial terms for $\text{year}_i$ in the first and second columns. By creating this set of regressors in the $\mathbf{Z}-$matrix you set -up an ordinary least squares statistical model. Some clever -NumPy array features will build $\mathbf{Z}$ +up an ordinary least squares statistical model. -1. `year[:,np.newaxis]` : takes the 1D array with shape `(179,)` and turns it into a 2D column vector with shape `(179,1)` -2. `**[1, 0]` : stacks two columns, in the first column is `year**1` and the second column is `year**0 == 1` +`Z` is a linear model with two parameters, i.e. a polynomial with degree `1`. +Therefore we can represent the model with `numpy.polynomial.Polynomial` and +use the fitting functionality to determine the model parameters: ```{code-cell} -Z = year[:, np.newaxis] ** [1, 0] +model = np.polynomial.Polynomial.fit(year, yi, deg=1) ``` -Now that you have the created a matrix of regressors, $\mathbf{Z},$ and -the observations are in vector, $\mathbf{y},$ you can use these -variables to build the an ordinary least squares model with -[`sm.OLS`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html). +By default, `Polynomial.fit` performs the fit in the domain determined by the +independent variable (`year` in this case). +The coefficients for the unscaled and unshifted model can be recovered with the +`convert` method: -```{code-cell} -model = sm.OLS(yi, Z) -``` - -Now, you can view the fitting constants, $A$ and $B$, and their standard -errors. Run the -[`fit`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html) and print the -[`summary`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.RegressionResults.summary.html) to view results as such, ```{code-cell} -results = model.fit() -print(results.summary()) +model = model.convert() +model ``` -The __OLS Regression Results__ summary gives a lot of information about -the regressors, $\mathbf{Z},$ and observations, $\mathbf{y}.$ The most -important outputs for your current analysis are - -``` -================================= - coef std err ---------------------------------- -x1 0.3416 0.006 -const -666.3264 11.890 -================================= -``` -where `x1` is slope, $A=0.3416$, `const` is the intercept, -$B=-666.364$, and `std error` gives the precision of constants -$A=0.342\pm 0.006~\dfrac{\log(\text{transistors}/\text{chip})}{\text{years}}$ and $B=-666\pm -12~\log(\text{transistors}/\text{chip}),$ where the units are in -$\log(\text{transistors}/\text{chip})$. You created an exponential growth model. -To get the constants, save them to an array `AB` with -`results.params` and assign $A$ and $B$ to `x1` and `constant`. +The individual parameters $A$ and $B$ are the coefficients of our linear model: ```{code-cell} -AB = results.params -A = AB[0] -B = AB[1] +B, A = model ``` Did manufacturers double the transistor count every two years? You have @@ -277,24 +246,14 @@ $\dfrac{\text{transistor_count}(\text{year} +2)}{\text{transistor_count}(\text{y \dfrac{e^{B}e^{A( \text{year} + 2)}}{e^{B}e^{A \text{year}}} = e^{2A}$ where increase in number of transistors is $xFactor,$ number of years is -2, and $A$ is the best fit slope on the semilog function. The error in -your -prediction, $\Delta(xFactor),$ comes from the precision of your constant -$A,$ which you calculated as the standard error $\Delta A= 0.006$. - -$\Delta (xFactor) = \frac{\partial}{\partial A}(e^{2A})\Delta A = 2Ae^{2A}\Delta A$ +2, and $A$ is the best fit slope on the semilog function. ```{code-cell} -print("Rate of semiconductors added on a chip every 2 years:") -print( - "\tx{:.2f} +/- {:.2f} semiconductors per chip".format( - np.exp((A) * 2), 2 * A * np.exp(2 * A) * 0.006 - ) -) +print(f"Rate of semiconductors added on a chip every 2 years: {np.exp(2 * A):.2f}") ``` Based upon your least-squares regression model, the number of -semiconductors per chip increased by a factor of $1.98\pm 0.01$ every two +semiconductors per chip increased by a factor of $1.98$ every two years. You have a model that predicts the number of semiconductors each year. Now compare your model to the actual manufacturing reports. Plot the linear regression results and all of the transistor counts. @@ -455,7 +414,7 @@ np.savez( transistor_count=transistor_count, transistor_count_predicted=transistor_count_predicted, transistor_Moores_law=transistor_Moores_law, - regression_csts=AB, + regression_csts=(A, B), ) ``` diff --git a/environment.yml b/environment.yml index 8421695f..137b04ca 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,6 @@ dependencies: - scipy - matplotlib - pandas - - statsmodels - imageio # For building the site - sphinx diff --git a/requirements.txt b/requirements.txt index 63b2eefa..0ec30181 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ numpy scipy matplotlib pandas -statsmodels imageio # For supporting .md-based notebooks jupytext diff --git a/tox.ini b/tox.ini index b14c817d..e6f26ac8 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,6 @@ deps = oldestdeps: matplotlib==3.4 oldestdeps: scipy==1.6 oldestdeps: pandas==1.2 - oldestdeps: statsmodels==0.13 allowlist_externals = bash @@ -28,7 +27,6 @@ commands = devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple scipy devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple matplotlib devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple pandas - devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple statsmodels pip freeze From a870f27b8f5c71313801f7c0a870f51090c49e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 12 Jan 2024 23:13:35 +0100 Subject: [PATCH 040/152] MAINT: update minimum required versions and CI maintanance (#199) * CI: update minimum versions following SPEC0 as of Q4 2023 * CI: adding python 3.12 to testing * MAINT: use the scientific python extra pip index for dev wheels Keep reinstalling numpy with a separated pip command to ensure the dev version is picked up even if a downstream dependency upper limited it * MAINT: content fixes for numpy 2.0 --- .github/workflows/ci_tests_run_notebooks.yml | 13 +++++++---- content/tutorial-ma.md | 6 ++--- tox.ini | 24 ++++++++++++-------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 9b55c58f..31ddb5d8 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -31,14 +31,19 @@ jobs: os: ubuntu-latest - python-version: '3.11' - toxenv: py311-test-devdeps - name: with Python 3.11 and developer versioned dependencies + toxenv: py311-test + name: with Python 3.11 and latest released version of dependencies + os: ubuntu-latest + + - python-version: '3.12' + toxenv: py312-test-devdeps + name: with Python 3.12 and developer versioned dependencies os: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/content/tutorial-ma.md b/content/tutorial-ma.md index 90a7fa31..fcb69eb2 100644 --- a/content/tutorial-ma.md +++ b/content/tutorial-ma.md @@ -92,7 +92,7 @@ rows of this file, since they contain other data we are not interested in. Separ # Read just the dates for columns 4-18 from the first row dates = np.genfromtxt( filename, - dtype=np.unicode_, + dtype=np.str_, delimiter=",", max_rows=1, usecols=range(4, 18), @@ -102,7 +102,7 @@ dates = np.genfromtxt( # columns, skipping the first six rows locations = np.genfromtxt( filename, - dtype=np.unicode_, + dtype=np.str_, delimiter=",", skip_header=6, usecols=(0, 1), @@ -119,7 +119,7 @@ nbcases = np.genfromtxt( ) ``` -Included in the `numpy.genfromtxt` function call, we have selected the [numpy.dtype](https://numpy.org/devdocs/reference/generated/numpy.dtype.html#numpy.dtype) for each subset of the data (either an integer - `numpy.int_` - or a string of characters - `numpy.unicode_`). We have also used the `encoding` argument to select `utf-8-sig` as the encoding for the file (read more about encoding in the [official Python documentation](https://docs.python.org/3/library/codecs.html#encodings-and-unicode). You can read more about the `numpy.genfromtxt` function from the [Reference Documentation](https://numpy.org/devdocs/reference/generated/numpy.genfromtxt.html#numpy.genfromtxt) or from the [Basic IO tutorial](https://numpy.org/devdocs/user/basics.io.genfromtxt.html). +Included in the `numpy.genfromtxt` function call, we have selected the [numpy.dtype](https://numpy.org/devdocs/reference/generated/numpy.dtype.html#numpy.dtype) for each subset of the data (either an integer - `numpy.int_` - or a string of characters - `numpy.str_`). We have also used the `encoding` argument to select `utf-8-sig` as the encoding for the file (read more about encoding in the [official Python documentation](https://docs.python.org/3/library/codecs.html#encodings-and-unicode). You can read more about the `numpy.genfromtxt` function from the [Reference Documentation](https://numpy.org/devdocs/reference/generated/numpy.genfromtxt.html#numpy.genfromtxt) or from the [Basic IO tutorial](https://numpy.org/devdocs/user/basics.io.genfromtxt.html). +++ diff --git a/tox.ini b/tox.ini index e6f26ac8..4c3157e3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{39,310,311}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} + py{39,310,311,312}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} requires = pip >= 19.3.1 @@ -8,6 +8,9 @@ requires = description = run tests +setenv = + devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple + deps = # We use these files to specify all the dependencies, and below we override # versions for specific testing schenarios @@ -15,18 +18,21 @@ deps = -rsite/requirements.txt -rrequirements.txt - oldestdeps: numpy==1.20 - oldestdeps: matplotlib==3.4 - oldestdeps: scipy==1.6 - oldestdeps: pandas==1.2 + oldestdeps: numpy==1.23 + oldestdeps: matplotlib==3.6 + oldestdeps: scipy==1.8 + oldestdeps: pandas==1.4 + + devdeps: numpy>=0.0.dev0 + devdeps: scipy>=0.0.dev0 + devdeps: matplotlib>=0.0.dev0 + devdeps: pandas>=0.0.dev0 allowlist_externals = bash commands = - devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple scipy - devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple matplotlib - devdeps: pip install -U --pre --only-binary :all: -i https://pypi.anaconda.org/scipy-wheels-nightly/simple pandas + # Force numpy reinstall to work around upper version limits in downstream dependencies (e.g. pandas) + devdeps: pip install -U --pre --no-deps --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy pip freeze From a660336f8e5b738c2700c889642470af22488f76 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 12 Jan 2024 15:11:51 -0800 Subject: [PATCH 041/152] Replace scipy.misc.face with scipy.datasets.face. --- content/tutorial-svd.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index a1fe60a4..6912d7a4 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -35,12 +35,12 @@ After this tutorial, you should be able to: ## Content -In this tutorial, we will use a [matrix decomposition](https://en.wikipedia.org/wiki/Matrix_decomposition) from linear algebra, the Singular Value Decomposition, to generate a compressed approximation of an image. We'll use the `face` image from the [scipy.misc](https://docs.scipy.org/doc/scipy/reference/misc.html#module-scipy.misc) module: +In this tutorial, we will use a [matrix decomposition](https://en.wikipedia.org/wiki/Matrix_decomposition) from linear algebra, the Singular Value Decomposition, to generate a compressed approximation of an image. We'll use the `face` image from the [scipy.datasets](https://docs.scipy.org/doc/scipy/reference/datasets.html) module: ```{code-cell} -from scipy import misc +from scipy import datasets -img = misc.face() +img = datasets.face() ``` **Note**: If you prefer, you can use your own image as you work through this tutorial. In order to transform your image into a NumPy array that can be manipulated, you can use the `imread` function from the [matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. Alternatively, you can use the [imageio.imread](https://imageio.readthedocs.io/en/stable/userapi.html#imageio.imread) function from the `imageio` library. Be aware that if you use your own image, you'll likely need to adapt the steps below. For more information on how images are treated when converted to NumPy arrays, see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. @@ -91,7 +91,7 @@ img[:, :, 0] ``` From the output above, we can see that every value in `img[:, :, 0]` is an integer value between 0 and 255, representing the level of red in each corresponding image pixel (keep in mind that this might be different if you -use your own image instead of [scipy.misc.face](https://docs.scipy.org/doc/scipy/reference/generated/scipy.misc.face.html#scipy.misc.face)). +use your own image instead of [scipy.datasets.face](https://docs.scipy.org/doc/scipy/reference/generated/scipy.datasets.face.html)). As expected, this is a 768x1024 matrix: From 684a08a782883c2e277956c6c3bf28bc39af9e5e Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 12 Jan 2024 15:12:24 -0800 Subject: [PATCH 042/152] Properly write ipython magic. --- content/save-load-arrays.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/save-load-arrays.md b/content/save-load-arrays.md index 434c7370..6768d938 100644 --- a/content/save-load-arrays.md +++ b/content/save-load-arrays.md @@ -127,7 +127,7 @@ print(load_xy.files) ``` ```{code-cell} -whos +%whos ``` ## Reassign the NpzFile arrays to `x` and `y` From 1dcfbf2c35121d7328b27dd5eaacdac003f8ed12 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 12 Jan 2024 15:15:16 -0800 Subject: [PATCH 043/152] Use string literals for LaTeX strings in fractals tutorial. --- content/tutorial-plotting-fractals.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/tutorial-plotting-fractals.md b/content/tutorial-plotting-fractals.md index e97b0cbe..373cd16f 100644 --- a/content/tutorial-plotting-fractals.md +++ b/content/tutorial-plotting-fractals.md @@ -301,14 +301,14 @@ For example, setting $c = \frac{\pi}{10}$ gives us a very elegant cloud shape, w ```{code-cell} ipython3 output = julia(mesh, c=np.pi/10, num_iter=20) -kwargs = {'title': 'f(z) = z^2 + \dfrac{\pi}{10}', 'cmap': 'plasma'} +kwargs = {'title': r'f(z) = z^2 + \dfrac{\pi}{10}', 'cmap': 'plasma'} plot_fractal(output, **kwargs); ``` ```{code-cell} ipython3 output = julia(mesh, c=-0.75 + 0.4j, num_iter=20) -kwargs = {'title': 'f(z) = z^2 - \dfrac{3}{4} + 0.4i', 'cmap': 'Greens_r'} +kwargs = {'title': r'f(z) = z^2 - \dfrac{3}{4} + 0.4i', 'cmap': 'Greens_r'} plot_fractal(output, **kwargs); ``` @@ -419,7 +419,7 @@ p.deriv() ```{code-cell} ipython3 output = newton_fractal(mesh, p, p.deriv(), num_iter=15, r=2) -kwargs = {'title': 'f(z) = z - \dfrac{(z^8 + 15z^4 - 16)}{(8z^7 + 60z^3)}', 'cmap': 'copper'} +kwargs = {'title': r'f(z) = z - \dfrac{(z^8 + 15z^4 - 16)}{(8z^7 + 60z^3)}', 'cmap': 'copper'} plot_fractal(output, **kwargs) ``` @@ -443,7 +443,7 @@ def d_tan(z): ```{code-cell} ipython3 output = newton_fractal(mesh, f_tan, d_tan, num_iter=15, r=50) -kwargs = {'title': 'f(z) = z - \dfrac{sin(z)cos(z)}{2}', 'cmap': 'binary'} +kwargs = {'title': r'f(z) = z - \dfrac{sin(z)cos(z)}{2}', 'cmap': 'binary'} plot_fractal(output, **kwargs); ``` From 5996bd6185b06ec0ebd18b49f64b6257d61de2bb Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 12 Jan 2024 16:41:50 -0800 Subject: [PATCH 044/152] Add optional pooch dep for scipy.datasets. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0ec30181..dd03cc89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # For the tutorials numpy scipy +pooch # for scipy.datasets matplotlib pandas imageio From 8fed7137cba7233a1c702dbf769e72f24e53ef08 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 12 Jan 2024 16:50:08 -0800 Subject: [PATCH 045/152] Handle SyntaxWarnings from invalid escape sequences. --- content/tutorial-plotting-fractals.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/content/tutorial-plotting-fractals.md b/content/tutorial-plotting-fractals.md index 373cd16f..a1921cea 100644 --- a/content/tutorial-plotting-fractals.md +++ b/content/tutorial-plotting-fractals.md @@ -334,7 +334,7 @@ def mandelbrot(mesh, num_iter=10, radius=2): ```{code-cell} ipython3 output = mandelbrot(mesh, num_iter=50) -kwargs = {'title': 'Mandelbrot \ set', 'cmap': 'hot'} +kwargs = {'title': 'Mandelbrot \\ set', 'cmap': 'hot'} plot_fractal(output, **kwargs); ``` @@ -370,8 +370,6 @@ for deg, ax in enumerate(axes.ravel()): diverge_len = general_julia(mesh, f=power, num_iter=15) ax.imshow(diverge_len, extent=[-2, 2, -2, 2], cmap='binary') ax.set_title(f'$f(z) = z^{degree} -1$') - -fig.tight_layout(); ``` Needless to say, there is a large amount of exploring that can be done by fiddling with the inputted function, value of $c$, number of iterations, radius and even the density of the mesh and choice of colours. @@ -475,7 +473,7 @@ We will denote this one 'Wacky fractal', as its equation would not be fun to try ```{code-cell} ipython3 output = newton_fractal(small_mesh, sin_sum, d_sin_sum, num_iter=10, r=1) -kwargs = {'title': 'Wacky \ fractal', 'figsize': (6, 6), 'extent': [-1, 1, -1, 1], 'cmap': 'terrain'} +kwargs = {'title': 'Wacky \\ fractal', 'figsize': (6, 6), 'extent': [-1, 1, -1, 1], 'cmap': 'terrain'} plot_fractal(output, **kwargs) ``` @@ -550,7 +548,7 @@ def accident(z): ```{code-cell} ipython3 output = general_julia(mesh, f=accident, num_iter=15, c=0, radius=np.pi) -kwargs = {'title': 'Accidental \ fractal', 'cmap': 'Blues'} +kwargs = {'title': 'Accidental \\ fractal', 'cmap': 'Blues'} plot_fractal(output, **kwargs); ``` From 8382ba04130f14aafe140dde0dbf655a5054b0c9 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 12 Jan 2024 20:54:36 -0800 Subject: [PATCH 046/152] Add pooch to conda deps too. --- content/x_y-squared.npz | Bin 670 -> 670 bytes environment.yml | 1 + 2 files changed, 1 insertion(+) diff --git a/content/x_y-squared.npz b/content/x_y-squared.npz index 6c32f1968769df4a983cb5b1006af0279ba4f009..e32d05c4b91f4b6b67416452fcdb58203176b834 100644 GIT binary patch delta 89 zcmbQoI*(N Date: Wed, 20 Dec 2023 16:53:10 -0800 Subject: [PATCH 047/152] CI: enabling cron and workflow dispatch for site building --- .github/workflows/conda.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 55d773fe..79d953c2 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -1,4 +1,4 @@ -name: Build site +name: Build site with conda on: push: @@ -7,6 +7,9 @@ on: pull_request: branches: - main + schedule: + - cron: '0 5 * * 1' + workflow_dispatch: jobs: test: @@ -21,7 +24,7 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true @@ -30,7 +33,7 @@ jobs: miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true - python-version: "3.10" + python-version: "3.11" auto-activate-base: false - name: inspect and build id: build_step From 8f6c272ce7ef1e98d9a91da069091e98a0f9f904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 20 Dec 2023 20:11:15 -0800 Subject: [PATCH 048/152] CI: trying newer miniconda --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 79d953c2..e3eb60a6 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true activate-environment: numpy-tutorials From 0164e427176a41682fbe144114cbd628dcf7e70d Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 19 Jan 2024 08:34:18 -0800 Subject: [PATCH 049/152] Try/except imports to deal with scipy datasets. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Brigitta Sipőcz --- content/tutorial-svd.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 6912d7a4..3798636a 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -38,9 +38,13 @@ After this tutorial, you should be able to: In this tutorial, we will use a [matrix decomposition](https://en.wikipedia.org/wiki/Matrix_decomposition) from linear algebra, the Singular Value Decomposition, to generate a compressed approximation of an image. We'll use the `face` image from the [scipy.datasets](https://docs.scipy.org/doc/scipy/reference/datasets.html) module: ```{code-cell} -from scipy import datasets +# TODO: Rm try-except with scipy 1.10 is the minimum supported version +try: + from scipy.datasets import face +except ImportError: # Data was in scipy.misc prior to scipy v1.10 + from scipy.misc import face -img = datasets.face() +img = face() ``` **Note**: If you prefer, you can use your own image as you work through this tutorial. In order to transform your image into a NumPy array that can be manipulated, you can use the `imread` function from the [matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. Alternatively, you can use the [imageio.imread](https://imageio.readthedocs.io/en/stable/userapi.html#imageio.imread) function from the `imageio` library. Be aware that if you use your own image, you'll likely need to adapt the steps below. For more information on how images are treated when converted to NumPy arrays, see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. From db5a326f43e7734262603bbd9277fbf651853b8c Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 19 Jan 2024 20:26:13 -0800 Subject: [PATCH 050/152] Revert changes to npz data file. --- content/x_y-squared.npz | Bin 670 -> 670 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/content/x_y-squared.npz b/content/x_y-squared.npz index e32d05c4b91f4b6b67416452fcdb58203176b834..6c32f1968769df4a983cb5b1006af0279ba4f009 100644 GIT binary patch delta 68 zcmbQoI**k#z?+#xWU@Sy_(Yj6(jp8Hpva(;*pP4m$N^z41`&qJiSlbF-(YlM7GVad KHU-n+Ozr^S*bV0Z delta 89 zcmbQoI*(N Date: Mon, 22 Jan 2024 11:56:56 -0800 Subject: [PATCH 051/152] CI: more descriptive job name [skip ci] --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index e3eb60a6..d94eaafb 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -1,4 +1,4 @@ -name: Build site with conda +name: Test building site with conda environment on: push: From d6d698bbfd2e2ff388460d627a18f10599293519 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 4 Mar 2024 11:52:34 -0800 Subject: [PATCH 052/152] Rm generated npz/csv files from tracking --- content/x_y-squared.csv | 11 ----------- content/x_y-squared.npz | Bin 670 -> 0 bytes 2 files changed, 11 deletions(-) delete mode 100644 content/x_y-squared.csv delete mode 100644 content/x_y-squared.npz diff --git a/content/x_y-squared.csv b/content/x_y-squared.csv deleted file mode 100644 index e74126ff..00000000 --- a/content/x_y-squared.csv +++ /dev/null @@ -1,11 +0,0 @@ -# x, y -0.000000000000000000e+00,0.000000000000000000e+00 -1.000000000000000000e+00,1.000000000000000000e+00 -2.000000000000000000e+00,4.000000000000000000e+00 -3.000000000000000000e+00,9.000000000000000000e+00 -4.000000000000000000e+00,1.600000000000000000e+01 -5.000000000000000000e+00,2.500000000000000000e+01 -6.000000000000000000e+00,3.600000000000000000e+01 -7.000000000000000000e+00,4.900000000000000000e+01 -8.000000000000000000e+00,6.400000000000000000e+01 -9.000000000000000000e+00,8.100000000000000000e+01 diff --git a/content/x_y-squared.npz b/content/x_y-squared.npz deleted file mode 100644 index 6c32f1968769df4a983cb5b1006af0279ba4f009..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 670 zcmWIWW@Zs#fB;1X8xBpr3qTGCb1{f8RKzD%WESh?6;v`Z2!KUFDnYU!FxfBEHz1Oc zp^TwgJteg`xk%kgLER?PLS08eJuSbeq$n{jKEEg>6(sJKm{Xhz6fe$5EJy|NH4F`O zG<6he703r%5LYlTFhUtjP?{M^vp{K9D9r|?*`YKCl;(ue0VrXilh}}e92S*BDlA}r zhxtta>PJZ^tpcSDp|k^(4usMH-i%Bl%(zkpR1uJB00%xIouKPO4Mk*qJV2d@Fb(i# R1xF7P0~135P?-ccG64-KTr>ax From 4d06bdcf31500301e09fe25bfa550ba5cc49a721 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 4 Mar 2024 11:58:28 -0800 Subject: [PATCH 053/152] Add generated files to gitignore. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a1654131..bee3f8ec 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,5 @@ content/tutorial-x-ray-image-processing/xray_image.gif content/video content/*ipynb content/tutorial-nlp-from-scratch/parameters.npy -content/tutorial-nlp-from-scratch/*ipynb \ No newline at end of file +content/tutorial-nlp-from-scratch/*ipynb +content/x_y-squared* From c3c581a4a5bf5e276902caa4a8837f57e54b892b Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 4 Mar 2024 12:01:51 -0800 Subject: [PATCH 054/152] Replace hard-coded file output with \!head --- content/save-load-arrays.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/content/save-load-arrays.md b/content/save-load-arrays.md index 6768d938..2620bfaf 100644 --- a/content/save-load-arrays.md +++ b/content/save-load-arrays.md @@ -187,18 +187,8 @@ np.savetxt("x_y-squared.csv", X=array_out, header="x, y", delimiter=",") Open the file, `x_y-squared.csv`, and you'll see the following: -``` -# x, y -0.000000000000000000e+00,0.000000000000000000e+00 -1.000000000000000000e+00,1.000000000000000000e+00 -2.000000000000000000e+00,4.000000000000000000e+00 -3.000000000000000000e+00,9.000000000000000000e+00 -4.000000000000000000e+00,1.600000000000000000e+01 -5.000000000000000000e+00,2.500000000000000000e+01 -6.000000000000000000e+00,3.600000000000000000e+01 -7.000000000000000000e+00,4.900000000000000000e+01 -8.000000000000000000e+00,6.400000000000000000e+01 -9.000000000000000000e+00,8.100000000000000000e+01 +```{code-cell} +!head x_y-squared.csv ``` ## Our arrays as a csv file From 2ab567faabadadcceaea3e01ae6fb084d822517b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 23 May 2024 11:07:59 -0700 Subject: [PATCH 055/152] CI: adding predeps job (#207) --- .github/workflows/ci_tests_run_notebooks.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 31ddb5d8..bb561926 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -35,6 +35,11 @@ jobs: name: with Python 3.11 and latest released version of dependencies os: ubuntu-latest + - python-version: '3.12' + toxenv: py312-test-predeps + name: with Python 3.12 and latest or pre-release version of dependencies + os: ubuntu-latest + - python-version: '3.12' toxenv: py312-test-devdeps name: with Python 3.12 and developer versioned dependencies From 1280f4b6aecc740cf88dfef998f3c6af291d3ca9 Mon Sep 17 00:00:00 2001 From: partev Date: Sun, 2 Jun 2024 11:45:28 -0400 Subject: [PATCH 056/152] DOC: Update tutorial-svd.md fix a typo fix a typo in a link to Wikipedia article --- content/tutorial-svd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 3798636a..614a7dc5 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -154,7 +154,7 @@ $$U \Sigma V^T = A$$ where $U$ and $V^T$ are square and $\Sigma$ is the same size as $A$. $\Sigma$ is a diagonal matrix and contains the [singular values](https://en.wikipedia.org/wiki/Singular_value) of $A$, organized from largest to smallest. These values are always non-negative and can be used as an indicator of the "importance" of some features represented by the matrix $A$. -Let's see how this works in practice with just one matrix first. Note that according to [colorimetry](https://en.wikipedia.org/wiki/Grayscale#Colorimetric_(perceptual_luminance-reserving)_conversion_to_grayscale), +Let's see how this works in practice with just one matrix first. Note that according to [colorimetry](https://en.wikipedia.org/wiki/Grayscale#Colorimetric_(perceptual_luminance-preserving)_conversion_to_grayscale), it is possible to obtain a fairly reasonable grayscale version of our color image if we apply the formula $$Y = 0.2126 R + 0.7152 G + 0.0722 B$$ From 259cf94d1c20ccf79e025debe21ae9ca97383814 Mon Sep 17 00:00:00 2001 From: partev Date: Sun, 2 Jun 2024 12:43:32 -0400 Subject: [PATCH 057/152] Update tutorial-nlp-from-scratch.md remove reference to MXNet which has been deprecated. https://lists.apache.org/thread/vzcy47wrbf89nljokghjqgzn0loq7knc --- content/tutorial-nlp-from-scratch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index 68a31d27..e51d6b1d 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -1051,7 +1051,7 @@ To further enhance and optimize your neural network model, you can consider one Nowadays, LSTMs have been replaced by the [Transformer](https://jalammar.github.io/illustrated-transformer/)( which uses [Attention](https://jalammar.github.io/visualizing-neural-machine-translation-mechanics-of-seq2seq-models-with-attention/) to tackle all the problems that plague an LSTM such as as lack of [transfer learning](https://en.wikipedia.org/wiki/Transfer_learning), lack of [parallel training](https://web.stanford.edu/~rezab/classes/cme323/S16/projects_reports/hedge_usmani.pdf) and a long gradient chain for lengthy sequences -Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as PyTorch, JAX, TensorFlow or MXNet — that provide NumPy-like APIs, have built-in automatic differentiation and GPU support, and are designed for high-performance numerical computing and machine learning. +Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as PyTorch, JAX or TensorFlow — that provide NumPy-like APIs, have built-in automatic differentiation and GPU support, and are designed for high-performance numerical computing and machine learning. Finally, to know more about how ethics come into play when developing a machine learning model, you can refer to the following resources : - Data ethics resources by the Turing Institute. https://www.turing.ac.uk/research/data-ethics From af927e31e9c87a41dcc8c7dea963839e87ae3c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 4 Jun 2024 09:55:37 -0700 Subject: [PATCH 058/152] Using hashes for all actions (#210) --- .github/workflows/ci_tests_run_notebooks.yml | 4 ++-- .github/workflows/circleci.yml | 2 +- .github/workflows/conda.yml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index bb561926..ba47d458 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -46,9 +46,9 @@ jobs: os: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d #v5.1.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index be39709f..ee793c73 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -5,7 +5,7 @@ jobs: name: Run CircleCI artifacts redirector steps: - name: GitHub Action step - uses: larsoner/circleci-artifacts-redirector-action@master + uses: larsoner/circleci-artifacts-redirector-action@4e13a10d89177f4bfc8007a7064bdbeda848d8d1 # v1.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} api-token: ${{ secrets.CIRCLE_TOKEN }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index d94eaafb..03a0b4a0 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -24,8 +24,8 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@v4 - - uses: conda-incubator/setup-miniconda@v3 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: auto-update-conda: true activate-environment: numpy-tutorials @@ -43,7 +43,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: sphinx-build-artifact path: site/_build/html/reports From 5ae3d377898e078ad341f73c986b05bee8492639 Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Wed, 5 Jun 2024 10:45:24 +0200 Subject: [PATCH 059/152] =?UTF-8?q?Remove=20likely=20copy-paste=20of=20pil?= =?UTF-8?q?crow=20("=C2=B6")?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- content/tutorial-nlp-from-scratch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index e51d6b1d..2eb581da 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -441,7 +441,7 @@ emb_path = textproc.unzipper(glove, 'glove.6B.300d.txt') emb_matrix = textproc.loadGloveModel(emb_path) ``` -## 3. Build the Deep Learning Model¶ +## 3. Build the Deep Learning Model It is time to start implementing our LSTM! You will have to first familiarize yourself with some high-level concepts of the basic building blocks of a deep learning model. You can refer to the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) for the same. You will then learn how a Recurrent Neural Network differs from a plain Neural Network and what makes it so suitable for processing sequential data. Afterwards, you will construct the building blocks of a simple deep learning model in Python and NumPy and train it to learn to classify the sentiment of a piece of text as positive or negative with a certain level of accuracy From be5acf1f9fda9f893cc9e30ae95c9f50d86353d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 14 Jun 2024 02:15:48 -0700 Subject: [PATCH 060/152] Adding dependabot for GHA version updates (#211) --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..4ebfc460 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: ".github/workflows" + schedule: + interval: "weekly" + groups: + actions: + patterns: + - "*" + labels: + - "infrastructure" From b19d28f8d6fb61564f28623a982d3f7cee5f220e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 09:16:52 +0000 Subject: [PATCH 061/152] Bump actions/checkout in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 4.1.6 to 4.1.7 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/a5ac7e51b41094c92402da3b24376905380afc29...692973e3d937129bcbf40652eb9f2f61becf3332) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/conda.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index ba47d458..9f0c991a 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -46,7 +46,7 @@ jobs: os: ubuntu-latest steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d #v5.1.0 with: diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 03a0b4a0..c4860014 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -24,7 +24,7 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: auto-update-conda: true From 694d57c6b24c09d372dbaa4872cb031c9a6c90f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 17 Jun 2024 17:45:26 -0700 Subject: [PATCH 062/152] CI: update action that got moved org --- .github/workflows/circleci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index ee793c73..88ac6e4f 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -5,7 +5,7 @@ jobs: name: Run CircleCI artifacts redirector steps: - name: GitHub Action step - uses: larsoner/circleci-artifacts-redirector-action@4e13a10d89177f4bfc8007a7064bdbeda848d8d1 # v1.0.0 + uses: scientific-python/circleci-artifacts-redirector-action@4e13a10d89177f4bfc8007a7064bdbeda848d8d1 # v1.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} api-token: ${{ secrets.CIRCLE_TOKEN }} From 49b33035aa7a7fe1430f145e6ee65a77624b8cb7 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Tue, 18 Jun 2024 10:00:38 -0700 Subject: [PATCH 063/152] Add Jupytext requirement for binder. (#217) I think binder looks at env.yaml not at requirements.txt maybe that's the reason that binder don't have jupytext installed --- environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 509d1bc7..15dd1085 100644 --- a/environment.yml +++ b/environment.yml @@ -14,3 +14,5 @@ dependencies: - myst-nb - sphinx-book-theme - sphinx-copybutton + # to load the md files in binder + - jupytext From ce12babf8b4f00e6ca030c17f0d729563daee14f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:34:42 +0000 Subject: [PATCH 064/152] Bump actions/upload-artifact in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/upload-artifact` from 4.3.3 to 4.3.4 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/65462800fd760344b1a7b4382951275a0abb4808...0b2256b8c012f0828dc542b3febcab082c67f72b) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index c4860014..24702096 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -43,7 +43,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 with: name: sphinx-build-artifact path: site/_build/html/reports From e1510d893552bdd4548a7eae118ff05dcd638857 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:29:33 +0000 Subject: [PATCH 065/152] Bump the actions group across 1 directory with 2 updates Bumps the actions group with 2 updates in the /.github/workflows directory: [actions/setup-python](https://github.com/actions/setup-python) and [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/setup-python` from 5.1.0 to 5.1.1 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/82c7e631bb3cdc910f68e0081d67478d79c6982d...39cd14951b08e74b54015e9e001cdefcf80e669f) Updates `actions/upload-artifact` from 4.3.4 to 4.3.5 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/0b2256b8c012f0828dc542b3febcab082c67f72b...89ef406dd8d7e03cfd12d9e0a4a378f454709029) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/conda.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 9f0c991a..18858f65 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -48,7 +48,7 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d #v5.1.0 + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f #v5.1.1 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 24702096..787b626e 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -43,7 +43,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 with: name: sphinx-build-artifact path: site/_build/html/reports From 974b89e190c8a62f38ddeaac1d35135929959903 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 5 Aug 2024 12:32:41 -0700 Subject: [PATCH 066/152] Fix typo in Moore's law tutorial. --- content/mooreslaw-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/mooreslaw-tutorial.md b/content/mooreslaw-tutorial.md index b9fefc8d..8e66104e 100644 --- a/content/mooreslaw-tutorial.md +++ b/content/mooreslaw-tutorial.md @@ -183,7 +183,7 @@ print("trans. cnt:\t", transistor_count[:10]) You are creating a function that predicts the transistor count given a year. You have an _independent variable_, `year`, and a _dependent -variable_, `transistor_count`. Transform the independent variable to +variable_, `transistor_count`. Transform the dependent variable to log-scale, $y_i = \log($ `transistor_count[i]` $),$ From a379973e6fb1b67a0276f889ff2492142dc6db2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 12:42:19 +0000 Subject: [PATCH 067/152] Bump the actions group across 1 directory with 2 updates Bumps the actions group with 2 updates in the /.github/workflows directory: [actions/setup-python](https://github.com/actions/setup-python) and [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/setup-python` from 5.1.1 to 5.2.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/39cd14951b08e74b54015e9e001cdefcf80e669f...f677139bbe7f9c59b41e40162b753c062f5d49a3) Updates `actions/upload-artifact` from 4.3.5 to 4.4.0 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/89ef406dd8d7e03cfd12d9e0a4a378f454709029...50769540e7f4bd5e21e526ee35c689e35e0d6874) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/conda.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 18858f65..d7b6fbd8 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -48,7 +48,7 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f #v5.1.1 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 787b626e..d2f6cd86 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -43,7 +43,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: sphinx-build-artifact path: site/_build/html/reports From a8f7b4c49d0284b1e0ba8df7013d7ef161f455df Mon Sep 17 00:00:00 2001 From: Liam Connors Date: Thu, 19 Sep 2024 22:05:51 -0400 Subject: [PATCH 068/152] update broken link --- content/tutorial-svd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 614a7dc5..aded8df0 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -47,7 +47,7 @@ except ImportError: # Data was in scipy.misc prior to scipy v1.10 img = face() ``` -**Note**: If you prefer, you can use your own image as you work through this tutorial. In order to transform your image into a NumPy array that can be manipulated, you can use the `imread` function from the [matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. Alternatively, you can use the [imageio.imread](https://imageio.readthedocs.io/en/stable/userapi.html#imageio.imread) function from the `imageio` library. Be aware that if you use your own image, you'll likely need to adapt the steps below. For more information on how images are treated when converted to NumPy arrays, see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. +**Note**: If you prefer, you can use your own image as you work through this tutorial. In order to transform your image into a NumPy array that can be manipulated, you can use the `imread` function from the [matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. Alternatively, you can use the [imageio.imread](https://imageio.readthedocs.io/en/stable/_autosummary/imageio.v3.imread.html) function from the `imageio` library. Be aware that if you use your own image, you'll likely need to adapt the steps below. For more information on how images are treated when converted to NumPy arrays, see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. +++ From 01b934cf47ac5499bfaf6c872419921891f7055e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:40:26 +0000 Subject: [PATCH 069/152] Bump actions/checkout in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 4.1.7 to 4.2.0 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/692973e3d937129bcbf40652eb9f2f61becf3332...d632683dd7b4114ad314bca15554477dd762a938) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/conda.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index d7b6fbd8..e69d8069 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -46,7 +46,7 @@ jobs: os: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0 with: diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index d2f6cd86..4be1c835 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -24,7 +24,7 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: auto-update-conda: true From f6a2721f6324ecb13771640f4905b068e6cf390a Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 1 Oct 2024 13:26:50 -0700 Subject: [PATCH 070/152] Turn off edit-this-page button. --- site/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/conf.py b/site/conf.py index e1fe5b63..4aea25ca 100644 --- a/site/conf.py +++ b/site/conf.py @@ -65,7 +65,7 @@ "repository_branch": "main", "use_repository_button": True, "use_issues_button": True, - "use_edit_page_button": True, + "use_edit_page_button": False, "path_to_docs": "site/", "launch_buttons": { "binderhub_url": "https://mybinder.org", From 0dce8f3dc15623504b5d0be0015a29471dd4a594 Mon Sep 17 00:00:00 2001 From: Liam Connors Date: Tue, 1 Oct 2024 22:30:25 -0400 Subject: [PATCH 071/152] fix broken link --- content/mooreslaw-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/mooreslaw-tutorial.md b/content/mooreslaw-tutorial.md index 8e66104e..93722261 100644 --- a/content/mooreslaw-tutorial.md +++ b/content/mooreslaw-tutorial.md @@ -22,7 +22,7 @@ In 1965, engineer Gordon Moore [predicted](https://en.wikipedia.org/wiki/Moore%27s_law) that transistors on a chip would double every two years in the coming decade [[1](https://en.wikipedia.org/wiki/Moore%27s_law), -[2](https://newsroom.intel.com/wp-content/uploads/sites/11/2018/05/moores-law-electronics.pdf)]. +[2](https://www.intel.com/content/www/us/en/newsroom/resources/moores-law.html)]. You'll compare Moore's prediction against actual transistor counts in the 53 years following his prediction. You will determine the best-fit constants to describe the exponential growth of transistors on semiconductors compared to Moore's Law. From 3d073169119f391f65e727256ffad834c3ba0c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melissa=20Weber=20Mendon=C3=A7a?= Date: Wed, 2 Oct 2024 16:23:07 -0300 Subject: [PATCH 072/152] Fix conda job in CI --- .github/workflows/conda.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 4be1c835..b285d395 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -30,7 +30,6 @@ jobs: auto-update-conda: true activate-environment: numpy-tutorials environment-file: environment.yml - miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true python-version: "3.11" From 350dc6c1e02d1e6226316bf505d95178b3948a4c Mon Sep 17 00:00:00 2001 From: Liam Connors Date: Mon, 7 Oct 2024 19:13:58 -0400 Subject: [PATCH 073/152] remove broken reference --- content/mooreslaw-tutorial.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/content/mooreslaw-tutorial.md b/content/mooreslaw-tutorial.md index 93722261..42c0de15 100644 --- a/content/mooreslaw-tutorial.md +++ b/content/mooreslaw-tutorial.md @@ -21,8 +21,7 @@ _The number of transistors reported per a given chip plotted on a log scale in t In 1965, engineer Gordon Moore [predicted](https://en.wikipedia.org/wiki/Moore%27s_law) that transistors on a chip would double every two years in the coming decade -[[1](https://en.wikipedia.org/wiki/Moore%27s_law), -[2](https://www.intel.com/content/www/us/en/newsroom/resources/moores-law.html)]. +[[1](https://en.wikipedia.org/wiki/Moore%27s_law)]. You'll compare Moore's prediction against actual transistor counts in the 53 years following his prediction. You will determine the best-fit constants to describe the exponential growth of transistors on semiconductors compared to Moore's Law. @@ -130,7 +129,7 @@ print("This is x{:.2f} more transistors than 1971".format(ML_1973 / ML_1971)) Now, make a prediction based upon the historical data for semiconductors per chip. The [Transistor Count -\[4\]](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) +\[3\]](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) each year is in the `transistor_data.csv` file. Before loading a \*.csv file into a NumPy array, its a good idea to inspect the structure of the file first. Then, locate the columns of interest and save them to a @@ -520,7 +519,7 @@ double every two years from 1965 through 1975, but the average growth has maintained a consistent increase of $\times 1.98 \pm 0.01$ every two years from 1971 through 2019. In 2015, Moore revised his prediction to say Moore's law should hold until 2025. -[[3](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress)]. +[[2](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress)]. You can share these results as a zipped NumPy array file, `mooreslaw_regression.npz`, or as another csv, `mooreslaw_regression.csv`. The amazing progress in semiconductor @@ -533,6 +532,5 @@ has been over the last half-century. ## References 1. ["Moore's Law." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Moore%27s_law) -2. [Moore, Gordon E. (1965-04-19). "Cramming more components onto integrated circuits". intel.com. Electronics Magazine. Retrieved April 1, 2020.](https://newsroom.intel.com/wp-content/uploads/sites/11/2018/05/moores-law-electronics.pdf) -3. [Courtland, Rachel. "Gordon Moore: The Man Whose Name Means Progress." IEEE Spectrum. 30 Mar. 2015.](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress). -4. ["Transistor Count." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) +2. [Courtland, Rachel. "Gordon Moore: The Man Whose Name Means Progress." IEEE Spectrum. 30 Mar. 2015.](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress). +3. ["Transistor Count." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) From 147ded20e8fdf6be89988a0f9ee0b0a4cde0ea57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:18:02 +0000 Subject: [PATCH 074/152] Bump the actions group in /.github/workflows with 2 updates Bumps the actions group in /.github/workflows with 2 updates: [actions/checkout](https://github.com/actions/checkout) and [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/checkout` from 4.2.0 to 4.2.1 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/d632683dd7b4114ad314bca15554477dd762a938...eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871) Updates `actions/upload-artifact` from 4.4.0 to 4.4.3 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/50769540e7f4bd5e21e526ee35c689e35e0d6874...b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/conda.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index e69d8069..71f23f81 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -46,7 +46,7 @@ jobs: os: ubuntu-latest steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0 with: diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index b285d395..a390cf22 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -24,7 +24,7 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: auto-update-conda: true @@ -42,7 +42,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: sphinx-build-artifact path: site/_build/html/reports From d770172911756291bdf87f3dbb38c70839f5dff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 14 Oct 2024 09:03:34 -0700 Subject: [PATCH 075/152] MAINT: doing only montly dependabot --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4ebfc460..8db2976b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "github-actions" directory: ".github/workflows" schedule: - interval: "weekly" + interval: "monthly" groups: actions: patterns: From d25c606607073db5b43855bff18716c2a3b4f6e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 12:53:51 +0000 Subject: [PATCH 076/152] Bump the actions group in /.github/workflows with 2 updates Bumps the actions group in /.github/workflows with 2 updates: [actions/checkout](https://github.com/actions/checkout) and [actions/setup-python](https://github.com/actions/setup-python). Updates `actions/checkout` from 4.2.1 to 4.2.2 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871...11bd71901bbe5b1630ceea73d27597364c9af683) Updates `actions/setup-python` from 5.2.0 to 5.3.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/f677139bbe7f9c59b41e40162b753c062f5d49a3...0b93645e9fea7318ecaed2b359559ac225c90a2b) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 4 ++-- .github/workflows/conda.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 71f23f81..a9c8bcee 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -46,9 +46,9 @@ jobs: os: ubuntu-latest steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 #v5.2.0 + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #v5.3.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index a390cf22..4867c4fc 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -24,7 +24,7 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: auto-update-conda: true From fa7a0e6bfe398e51958c4b71bf97279896099c1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Dec 2024 12:24:12 +0000 Subject: [PATCH 077/152] Bump conda-incubator/setup-miniconda Bumps the actions group in /.github/workflows with 1 update: [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda). Updates `conda-incubator/setup-miniconda` from 3.0.4 to 3.1.0 - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/a4260408e20b96e80095f42ff7f1a15b27dd94ca...d2e6a045a86077fb6cad6f5adf368e9076ddaa8d) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 4867c4fc..8ed5abb7 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 + - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 with: auto-update-conda: true activate-environment: numpy-tutorials From 510f847e073f1b408480a148a63966aaded74065 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:54:55 +0000 Subject: [PATCH 078/152] Bump actions/upload-artifact in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/upload-artifact` from 4.4.3 to 4.5.0 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882...6f51ac03b9356f520e9adb1b1b7802705f340c2b) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 8ed5abb7..a6ecde9e 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -42,7 +42,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: sphinx-build-artifact path: site/_build/html/reports From 552fcd08a574ede9fa1ce1e131d71edf73ebd523 Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Tue, 7 Jan 2025 14:23:23 -0800 Subject: [PATCH 079/152] Make some https URLs clickable (#241) --- content/tutorial-nlp-from-scratch.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index 2eb581da..ae2eeb14 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -107,8 +107,8 @@ We made sure to include different demographics in our data and included a range 1. **Text Denoising** : Before converting your text into vectors, it is important to clean it and remove all unhelpful parts a.k.a the noise from your data by converting all characters to lowercase, removing html tags, brackets and stop words (words that don't add much meaning to a sentence). Without this step the dataset is often a cluster of words that the computer doesn't understand. -2. **Converting words to vectors** : A word embedding is a learned representation for text where words that have the same meaning have a similar representation. Individual words are represented as real-valued vectors in a predefined vector space. GloVe is an unsupervised algorithm developed by Stanford for generating word embeddings by generating global word-word co-occurence matrix from a corpus. You can download the zipped files containing the embeddings from https://nlp.stanford.edu/projects/glove/. Here you can choose any of the four options for different sizes or training datasets. We have chosen the least memory consuming embedding file. - >The GloVe word embeddings include sets that were trained on billions of tokens, some up to 840 billion tokens. These algorithms exhibit stereotypical biases, such as gender bias which can be traced back to the original training data. For example certain occupations seem to be more biased towards a particular gender, reinforcing problematic stereotypes. The nearest solution to this problem are some de-biasing algorithms as the one presented in https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1184/reports/6835575.pdf which one can use on embeddings of their choice to mitigate bias, if present. +2. **Converting words to vectors** : A word embedding is a learned representation for text where words that have the same meaning have a similar representation. Individual words are represented as real-valued vectors in a predefined vector space. GloVe is an unsupervised algorithm developed by Stanford for generating word embeddings by generating global word-word co-occurence matrix from a corpus. You can download the zipped files containing the embeddings from [the GloVe official website](https://nlp.stanford.edu/projects/glove/). Here you can choose any of the four options for different sizes or training datasets. We have chosen the least memory consuming embedding file. + >The GloVe word embeddings include sets that were trained on billions of tokens, some up to 840 billion tokens. These algorithms exhibit stereotypical biases, such as gender bias which can be traced back to the original training data. For example certain occupations seem to be more biased towards a particular gender, reinforcing problematic stereotypes. The nearest solution to this problem are some de-biasing algorithms as the one presented in [this research article](https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1184/reports/6835575.pdf), which one can use on embeddings of their choice to mitigate bias, if present. You'll start with importing the necessary packages to build our Deep Learning network. @@ -1049,11 +1049,11 @@ To further enhance and optimize your neural network model, you can consider one - Initialize weights using [Xavier Initialization](https://d2l.ai/chapter_multilayer-perceptrons/numerical-stability-and-init.html#xavier-initialization) to prevent vanishing/exploding gradients instead of initializing them randomly. - Replace LSTM with a [Bidirectional LSTM](https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks) to use both left and right context for predicting sentiment. -Nowadays, LSTMs have been replaced by the [Transformer](https://jalammar.github.io/illustrated-transformer/)( which uses [Attention](https://jalammar.github.io/visualizing-neural-machine-translation-mechanics-of-seq2seq-models-with-attention/) to tackle all the problems that plague an LSTM such as as lack of [transfer learning](https://en.wikipedia.org/wiki/Transfer_learning), lack of [parallel training](https://web.stanford.edu/~rezab/classes/cme323/S16/projects_reports/hedge_usmani.pdf) and a long gradient chain for lengthy sequences +Nowadays, LSTMs have been replaced by the [Transformer](https://jalammar.github.io/illustrated-transformer/) which uses [Attention](https://jalammar.github.io/visualizing-neural-machine-translation-mechanics-of-seq2seq-models-with-attention/) to tackle all the problems that plague an LSTM such as lack of [transfer learning](https://en.wikipedia.org/wiki/Transfer_learning), lack of [parallel training](https://web.stanford.edu/~rezab/classes/cme323/S16/projects_reports/hedge_usmani.pdf), and a long gradient chain for lengthy sequences. Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as PyTorch, JAX or TensorFlow — that provide NumPy-like APIs, have built-in automatic differentiation and GPU support, and are designed for high-performance numerical computing and machine learning. Finally, to know more about how ethics come into play when developing a machine learning model, you can refer to the following resources : -- Data ethics resources by the Turing Institute. https://www.turing.ac.uk/research/data-ethics +- [Data ethics resources](https://www.turing.ac.uk/research/data-ethics) by the Turing Institute - Considering how artificial intelligence shifts power, an [article](https://www.nature.com/articles/d41586-020-02003-2) and [talk](https://slideslive.com/38923453/the-values-of-machine-learning) by Pratyusha Kalluri - More ethics resources on [this blog post](https://www.fast.ai/2018/09/24/ai-ethics-resources/) by Rachel Thomas and the [Radical AI podcast](https://www.radicalai.org/) From 54a5cfa8fef50b9f989611274bf9304bd8456098 Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Mon, 20 Jan 2025 09:46:12 -0800 Subject: [PATCH 080/152] DOC: Update Vector Mechanics book link --- content/tutorial-static_equilibrium.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/tutorial-static_equilibrium.md b/content/tutorial-static_equilibrium.md index 0e8b82f7..f649e56e 100644 --- a/content/tutorial-static_equilibrium.md +++ b/content/tutorial-static_equilibrium.md @@ -263,7 +263,7 @@ print("Reaction moment =", M) ``` ### Another Example -Let's look at a slightly more complicated model. In this example you will be observing a beam with two cables and an applied force. This time you need to find both the tension in the cords and the reaction forces of the beam. *(Source: [Vector Mechanics for Engineers: Statics](https://www.mheducation.com/highered/product/vector-mechanics-engineers-statics-beer-johnston/M9780077687304.html), Problem 4.106)* +Let's look at a slightly more complicated model. In this example you will be observing a beam with two cables and an applied force. This time you need to find both the tension in the cords and the reaction forces of the beam. *(Source: [Vector Mechanics for Engineers: Statics and Dynamics](https://www.mheducation.com/highered/product/Vector-Mechanics-for-Engineers-Statics-and-Dynamics-Beer.html), Problem 4.106)* ![image.png](_static/problem4.png) @@ -387,5 +387,5 @@ This same process can be applied to kinetic problems or in any number of dimensi ### References -1. [Vector Mechanics for Engineers: Statics (Beer & Johnston & Mazurek)](https://www.mheducation.com/highered/product/vector-mechanics-engineers-statics-beer-johnston/M9780077687304.html) +1. [Vector Mechanics for Engineers: Statics and Dynamics (Beer & Johnston & Mazurek & et al.)](https://www.mheducation.com/highered/product/Vector-Mechanics-for-Engineers-Statics-and-Dynamics-Beer.html) 2. [NumPy Reference](https://numpy.org/doc/stable/reference/) From 43949b51163d946ff32234d4acfa04392a37857e Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Sun, 26 Jan 2025 18:59:01 -0800 Subject: [PATCH 081/152] DOC: Update the link to linear algebra tutorial (#243) --- content/tutorial-deep-learning-on-mnist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-deep-learning-on-mnist.md b/content/tutorial-deep-learning-on-mnist.md index a41438c5..fc874f02 100644 --- a/content/tutorial-deep-learning-on-mnist.md +++ b/content/tutorial-deep-learning-on-mnist.md @@ -33,7 +33,7 @@ This tutorial was adapted from the work by [Andrew Trask](https://github.com/iam The reader should have some knowledge of Python, NumPy array manipulation, and linear algebra. In addition, you should be familiar with main concepts of [deep learning](https://en.wikipedia.org/wiki/Deep_learning). -To refresh the memory, you can take the [Python](https://docs.python.org/dev/tutorial/index.html) and [Linear algebra on n-dimensional arrays](https://numpy.org/doc/stable/user/tutorial-svd.html) tutorials. +To refresh the memory, you can take the [Python](https://docs.python.org/dev/tutorial/index.html) and [Linear algebra on n-dimensional arrays](https://numpy.org/numpy-tutorials/content/tutorial-svd.html) tutorials. You are advised to read the [Deep learning](http://www.cs.toronto.edu/~hinton/absps/NatureDeepReview.pdf) paper published in 2015 by Yann LeCun, Yoshua Bengio, and Geoffrey Hinton, who are regarded as some of the pioneers of the field. You should also consider reading Andrew Trask's [Grokking Deep Learning](https://www.manning.com/books/grokking-deep-learning), which teaches deep learning with NumPy. From 4fbfeac3aea25a10290141467dd543b5a80aa065 Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Mon, 27 Jan 2025 16:41:19 -0800 Subject: [PATCH 082/152] DOC: Update link to Central Pollution Control Board of India --- content/tutorial-air-quality-analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-air-quality-analysis.md b/content/tutorial-air-quality-analysis.md index 14e46acf..fda0dbf3 100644 --- a/content/tutorial-air-quality-analysis.md +++ b/content/tutorial-air-quality-analysis.md @@ -97,7 +97,7 @@ With this, we have successfully imported the data and checked that it is complet ## Calculating the Air Quality Index -We will calculate the AQI using [the method](https://app.cpcbccr.com/ccr_docs/FINAL-REPORT_AQI_.pdf) adopted by the [Central Pollution Control Board](https://www.cpcb.nic.in/national-air-quality-index) of India. To summarize the steps: +We will calculate the AQI using [the method](https://app.cpcbccr.com/ccr_docs/FINAL-REPORT_AQI_.pdf) adopted by the [Central Pollution Control Board](https://www.cpcb.nic.in/national-air-quality-index/) of India. To summarize the steps: - Collect 24-hourly average concentration values for the standard pollutants; 8-hourly in case of CO and O3. From 2d2a30fc60da12ced652a5845b338a92793f6ddc Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Tue, 28 Jan 2025 22:04:31 -0800 Subject: [PATCH 083/152] DOC: Update another link to linear algebra tutorial (#245) --- content/tutorial-nlp-from-scratch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index ae2eeb14..a4771883 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -31,7 +31,7 @@ Today, Deep Learning is getting adopted in everyday life and now it is more impo ## Prerequisites -You are expected to be familiar with the Python programming language and array manipulation with NumPy. In addition, some understanding of Linear Algebra and Calculus is recommended. You should also be familiar with how Neural Networks work. For reference, you can visit the [Python](https://docs.python.org/dev/tutorial/index.html), [Linear algebra on n-dimensional arrays](https://numpy.org/doc/stable/user/tutorial-svd.html) and [Calculus](https://d2l.ai/chapter_appendix-mathematics-for-deep-learning/multivariable-calculus.html) tutorials. +You are expected to be familiar with the Python programming language and array manipulation with NumPy. In addition, some understanding of Linear Algebra and Calculus is recommended. You should also be familiar with how Neural Networks work. For reference, you can visit the [Python](https://docs.python.org/dev/tutorial/index.html), [Linear algebra on n-dimensional arrays](https://numpy.org/numpy-tutorials/content/tutorial-svd.html) and [Calculus](https://d2l.ai/chapter_appendix-mathematics-for-deep-learning/multivariable-calculus.html) tutorials. To get a refresher on Deep Learning basics, You should consider reading [the d2l.ai book](https://d2l.ai/chapter_recurrent-neural-networks/index.html), which is an interactive deep learning book with multi-framework code, math, and discussions. You can also go through the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) to understand how a basic neural network is implemented from scratch. From c5cd85f160d1549dbb66fc1be427d00f582e3b6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 12:49:08 +0000 Subject: [PATCH 084/152] Bump the actions group in /.github/workflows with 3 updates Bumps the actions group in /.github/workflows with 3 updates: [actions/setup-python](https://github.com/actions/setup-python), [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) and [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/setup-python` from 5.3.0 to 5.4.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/0b93645e9fea7318ecaed2b359559ac225c90a2b...42375524e23c412d93fb67b49958b491fce71c38) Updates `conda-incubator/setup-miniconda` from 3.1.0 to 3.1.1 - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/d2e6a045a86077fb6cad6f5adf368e9076ddaa8d...505e6394dae86d6a5c7fbb6e3fb8938e3e863830) Updates `actions/upload-artifact` from 4.5.0 to 4.6.0 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/6f51ac03b9356f520e9adb1b1b7802705f340c2b...65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/conda.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index a9c8bcee..243a59d0 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -48,7 +48,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #v5.3.0 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 #v5.4.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index a6ecde9e..cf78ba1e 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0 + - uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 with: auto-update-conda: true activate-environment: numpy-tutorials @@ -42,7 +42,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: sphinx-build-artifact path: site/_build/html/reports From 54bef6fe73ec03af907e27a498599bc746070e00 Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Sun, 2 Feb 2025 19:06:11 -0800 Subject: [PATCH 085/152] DOC: Update two links in deep reinforcement learning tutorial (#247) --- ...orial-deep-reinforcement-learning-with-pong-from-pixels.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md index 69607388..1598e572 100644 --- a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md +++ b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md @@ -22,7 +22,7 @@ Help improve this article by developing an example with reduced dependency footprint! ``` -This tutorial demonstrates how to implement a deep reinforcement learning (RL) agent from scratch using a policy gradient method that learns to play the [Pong](https://gym.openai.com/envs/Pong-v0/) video game using screen pixels as inputs with NumPy. Your Pong agent will obtain experience on the go using an [artificial neural network](https://en.wikipedia.org/wiki/Artificial_neural_network) as its [policy](https://en.wikipedia.org/wiki/Reinforcement_learning). +This tutorial demonstrates how to implement a deep reinforcement learning (RL) agent from scratch using a policy gradient method that learns to play the [Pong](https://en.wikipedia.org/wiki/Pong) video game using screen pixels as inputs with NumPy. Your Pong agent will obtain experience on the go using an [artificial neural network](https://en.wikipedia.org/wiki/Artificial_neural_network) as its [policy](https://en.wikipedia.org/wiki/Reinforcement_learning). Pong is a 2D game from 1972 where two players use "rackets" to play a form of table tennis. Each player moves the racket up and down the screen and tries to hit a ball in their opponent's direction by touching it. The goal is to hit the ball such that it goes past the opponent's racket (they miss their shot). According to the rules, if a player reaches 21 points, they win. In Pong, the RL agent that learns to play against an opponent is displayed on the right. @@ -32,7 +32,7 @@ This example is based on the [code](https://gist.github.com/karpathy/a4166c7fe25 ## Prerequisites -- **OpenAI Gym**: To help with the game environment, you will use [Gym](https://gym.openai.com) — an open-source Python interface [developed by OpenAI](https://arxiv.org/abs/1606.01540) that helps perform RL tasks while supporting many simulation environments. +- **OpenAI Gym**: To help with the game environment, you will use [Gym](https://github.com/openai/gym) — an open-source Python interface [developed by OpenAI](https://arxiv.org/abs/1606.01540) that helps perform RL tasks while supporting many simulation environments. - **Python and NumPy**: The reader should have some knowledge of Python, NumPy array manipulation, and linear algebra. - **Deep learning and deep RL**: You should be familiar with main concepts of [deep learning](https://en.wikipedia.org/wiki/Deep_learning), which are explained in the [Deep learning](http://www.cs.toronto.edu/~hinton/absps/NatureDeepReview.pdf) paper published in 2015 by Yann LeCun, Yoshua Bengio, and Geoffrey Hinton, who are regarded as some of the pioneers of the field. The tutorial will try to guide you through the main concepts of deep RL and you will find various literature with links to original sources for your convenience. - **Jupyter notebook environments**: Because RL experiments can require high computing power, you can run the tutorial on the cloud for free using [Binder](https://mybinder.org) or [Google Colaboratory](https://colab.research.google.com/notebooks/intro.ipynb) (which offers free limited GPU and TPU acceleration). From 3eadd491e3236fa6e87376b7a0410ca1ac92c422 Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Mon, 10 Feb 2025 18:33:15 -0800 Subject: [PATCH 086/152] DOC: Update the MNIST data link (#249) --- content/tutorial-deep-learning-on-mnist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-deep-learning-on-mnist.md b/content/tutorial-deep-learning-on-mnist.md index fc874f02..89647016 100644 --- a/content/tutorial-deep-learning-on-mnist.md +++ b/content/tutorial-deep-learning-on-mnist.md @@ -62,7 +62,7 @@ This tutorial can be run locally in an isolated environment, such as [Virtualenv ## 1. Load the MNIST dataset -In this section, you will download the zipped MNIST dataset files originally stored in [Yann LeCun's website](http://yann.lecun.com/exdb/mnist/). Then, you will transform them into 4 files of NumPy array type using built-in Python modules. Finally, you will split the arrays into training and test sets. +In this section, you will download the zipped MNIST dataset files originally developed by Yann LeCun's research team. (More details of the MNIST dataset are available on [Kaggle](https://www.kaggle.com/datasets/hojjatk/mnist-dataset).) Then, you will transform them into 4 files of NumPy array type using built-in Python modules. Finally, you will split the arrays into training and test sets. **1.** Define a variable to store the training/test image/label names of the MNIST dataset in a list: From 4cce9690d3cbf1dd898cebe5ee326cd4504ebee4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Mar 2025 12:28:49 +0000 Subject: [PATCH 087/152] Bump actions/upload-artifact in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/upload-artifact` from 4.6.0 to 4.6.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08...4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index cf78ba1e..b9524df9 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -42,7 +42,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: sphinx-build-artifact path: site/_build/html/reports From 30e9ae688d672b551e5ecf4669cbeab15294e7a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Apr 2025 12:12:12 +0000 Subject: [PATCH 088/152] Bump the actions group in /.github/workflows with 2 updates Bumps the actions group in /.github/workflows with 2 updates: [actions/setup-python](https://github.com/actions/setup-python) and [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/setup-python` from 5.4.0 to 5.5.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/42375524e23c412d93fb67b49958b491fce71c38...8d9ed9ac5c53483de85588cdf95a591a75ab9f55) Updates `actions/upload-artifact` from 4.6.1 to 4.6.2 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1...ea165f8d65b6e75b540449e92b4886f43607fa02) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/conda.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 243a59d0..baafd383 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -48,7 +48,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 #v5.4.0 + uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 #v5.5.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index b9524df9..f4dce2f5 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -42,7 +42,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: sphinx-build-artifact path: site/_build/html/reports From 4987e37f58cc944cf91fd2e1026ab781b7a76a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 8 Apr 2025 18:27:04 -0700 Subject: [PATCH 089/152] MAINT: fix link causing warning in html build --- site/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/contributing.md b/site/contributing.md index c35db184..e95774a5 100644 --- a/site/contributing.md +++ b/site/contributing.md @@ -42,7 +42,7 @@ You may notice our content is in markdown format (`.md` files). We review and host notebooks in the [MyST-NB](https://myst-nb.readthedocs.io/) format. We accept both Jupyter notebooks (`.ipynb`) and MyST-NB notebooks (`.md`). If you want to sync your `.ipynb` to your `.md` file follow the [pairing -tutorial](content/pairing.md). +tutorial](content/pairing). ```{toctree} :hidden: From 64b5f638943cd770db83dc6d03237c3db29441e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 8 Apr 2025 18:27:27 -0700 Subject: [PATCH 090/152] CI: adding make to tox allowlist --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4c3157e3..ff48327d 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ deps = devdeps: matplotlib>=0.0.dev0 devdeps: pandas>=0.0.dev0 -allowlist_externals = bash +allowlist_externals = bash, make commands = # Force numpy reinstall to work around upper version limits in downstream dependencies (e.g. pandas) From a553bed3464bee1b72b0d478fca0aa4629f715d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 May 2025 12:37:51 +0000 Subject: [PATCH 091/152] Bump actions/setup-python in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [actions/setup-python](https://github.com/actions/setup-python). Updates `actions/setup-python` from 5.5.0 to 5.6.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/8d9ed9ac5c53483de85588cdf95a591a75ab9f55...a26af69be951a213d495a4c3e4e4022e16d87065) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 5.6.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index baafd383..7b54b916 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -48,7 +48,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 #v5.5.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 #v5.6.0 with: python-version: ${{ matrix.python-version }} From 180dca7042bd77ef844e69200bbc05d4effe4fec Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 20 May 2025 09:31:51 -0700 Subject: [PATCH 092/152] MNT: Point mnist data download to ossci server. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Brigitta Sipőcz Co-authored-by: Melissa Weber Mendonça --- content/tutorial-deep-learning-on-mnist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-deep-learning-on-mnist.md b/content/tutorial-deep-learning-on-mnist.md index 89647016..e1085b91 100644 --- a/content/tutorial-deep-learning-on-mnist.md +++ b/content/tutorial-deep-learning-on-mnist.md @@ -99,7 +99,7 @@ import os data_dir = "../_data" os.makedirs(data_dir, exist_ok=True) -base_url = "https://github.com/rossbar/numpy-tutorial-data-mirror/blob/main/" +base_url = "https://ossci-datasets.s3.amazonaws.com/mnist/" for fname in data_sources.values(): fpath = os.path.join(data_dir, fname) From 396f038fc38f816b750c881cbc1093745d01ed3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Jun 2025 12:50:58 +0000 Subject: [PATCH 093/152] Bump scientific-python/circleci-artifacts-redirector-action Bumps the actions group in /.github/workflows with 1 update: [scientific-python/circleci-artifacts-redirector-action](https://github.com/scientific-python/circleci-artifacts-redirector-action). Updates `scientific-python/circleci-artifacts-redirector-action` from 1.0.0 to 1.1.0 - [Release notes](https://github.com/scientific-python/circleci-artifacts-redirector-action/releases) - [Commits](https://github.com/scientific-python/circleci-artifacts-redirector-action/compare/4e13a10d89177f4bfc8007a7064bdbeda848d8d1...7eafdb60666f57706a5525a2f5eb76224dc8779b) --- updated-dependencies: - dependency-name: scientific-python/circleci-artifacts-redirector-action dependency-version: 1.1.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/circleci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index 88ac6e4f..681c37aa 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -5,7 +5,7 @@ jobs: name: Run CircleCI artifacts redirector steps: - name: GitHub Action step - uses: scientific-python/circleci-artifacts-redirector-action@4e13a10d89177f4bfc8007a7064bdbeda848d8d1 # v1.0.0 + uses: scientific-python/circleci-artifacts-redirector-action@7eafdb60666f57706a5525a2f5eb76224dc8779b # v1.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} api-token: ${{ secrets.CIRCLE_TOKEN }} From 82031537432d1bb4426bd44e36b678c353cf1a7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:19:34 +0000 Subject: [PATCH 094/152] Bump conda-incubator/setup-miniconda Bumps the actions group in /.github/workflows with 1 update: [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda). Updates `conda-incubator/setup-miniconda` from 3.1.1 to 3.2.0 - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/505e6394dae86d6a5c7fbb6e3fb8938e3e863830...835234971496cad1653abb28a638a281cf32541f) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-version: 3.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index f4dce2f5..18da4854 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3.1.1 + - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 with: auto-update-conda: true activate-environment: numpy-tutorials From 2290d1f1b458dab79424a3c5c0be10b7f23e26eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 12:39:02 +0000 Subject: [PATCH 095/152] Bump scientific-python/circleci-artifacts-redirector-action Bumps the actions group in /.github/workflows with 1 update: [scientific-python/circleci-artifacts-redirector-action](https://github.com/scientific-python/circleci-artifacts-redirector-action). Updates `scientific-python/circleci-artifacts-redirector-action` from 1.1.0 to 1.2.0 - [Release notes](https://github.com/scientific-python/circleci-artifacts-redirector-action/releases) - [Commits](https://github.com/scientific-python/circleci-artifacts-redirector-action/compare/7eafdb60666f57706a5525a2f5eb76224dc8779b...839631420e45a08af893032e5a5e8843bf47e8ff) --- updated-dependencies: - dependency-name: scientific-python/circleci-artifacts-redirector-action dependency-version: 1.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/circleci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index 681c37aa..642252b7 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -5,7 +5,7 @@ jobs: name: Run CircleCI artifacts redirector steps: - name: GitHub Action step - uses: scientific-python/circleci-artifacts-redirector-action@7eafdb60666f57706a5525a2f5eb76224dc8779b # v1.1.0 + uses: scientific-python/circleci-artifacts-redirector-action@839631420e45a08af893032e5a5e8843bf47e8ff # v1.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} api-token: ${{ secrets.CIRCLE_TOKEN }} From 5cfaf6741e87839a6d128c4a5d7d46d627899b9a Mon Sep 17 00:00:00 2001 From: Stefanie Senger <91849487+StefanieSenger@users.noreply.github.com> Date: Fri, 8 Aug 2025 20:13:45 +0200 Subject: [PATCH 096/152] Slightly improved explanation of image shape in svd-tutorial.md (#260) --- content/tutorial-svd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index aded8df0..494a4d3b 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -80,7 +80,7 @@ First, let's check for the shape of the data in our array. Since this image is t img.shape ``` -The output is a [tuple](https://docs.python.org/dev/tutorial/datastructures.html#tut-tuples) with three elements, which means that this is a three-dimensional array. In fact, since this is a color image, and we have used the `imread` function to read it, the data is organized in three 2D arrays, representing color channels (in this case, red, green and blue - RGB). You can see this by looking at the shape above: it indicates that we have an array of 3 matrices, each having shape 768x1024. +The output is a [tuple](https://docs.python.org/dev/tutorial/datastructures.html#tut-tuples) with three elements, which means that this is a three-dimensional array. Since this is a color image, and we have used the `imread` function to read it, the data is organized as a 768×1024 grid of pixels, where each pixel contains 3 values representing color channels (red, green and blue - RGB). You can see this by looking at the shape, where the leftmost number corresponds to the outermost axis (image height), the middle number to the next axis (image width) and the rightmost number to the innermost axis (the color channels). Furthermore, using the `ndim` property of this array, we can see that From 37d9155bae581ecf36015c33cd81824e619a5f14 Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Mon, 11 Aug 2025 05:56:01 -0700 Subject: [PATCH 097/152] DOC: Update link to MyST Markdown format page (#261) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * DOC: Update link to MyST Markdown format page * Update pairing.md Co-authored-by: Melissa Weber Mendonça --------- Co-authored-by: Melissa Weber Mendonça --- content/pairing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/pairing.md b/content/pairing.md index 573f15d4..322fb462 100644 --- a/content/pairing.md +++ b/content/pairing.md @@ -38,7 +38,7 @@ keep your `.ipynb` in sync with the content on NumPy tutorials. The NumPy tutorials use [Jupytext](https://jupytext.readthedocs.io/en/latest/index.html) to convert your `.ipynb` file to [MyST -Markdown](https://github.com/mwouts/jupytext/blob/master/docs/formats.md#myst-markdown) +Markdown](https://jupytext.readthedocs.io/en/latest/formats-markdown.html) format. Jupyter notebooks are stored on your disk in a From 7b208c3a5483f6eb15712dc96c0936f6b323d336 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Sep 2025 21:04:10 +0000 Subject: [PATCH 098/152] Bump actions/checkout in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 4.2.2 to 5.0.0 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/11bd71901bbe5b1630ceea73d27597364c9af683...08c6903cd8c0fde910a37f88322edcfb5dd907a8) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 5.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/conda.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index 7b54b916..dfd48f58 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -46,7 +46,7 @@ jobs: os: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 #v5.6.0 with: diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 18da4854..5a3a9039 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -24,7 +24,7 @@ jobs: shell: bash -l {0} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 with: auto-update-conda: true From 0157aac4af78b27d218c85cb1dd45274f7561497 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 23 Sep 2025 12:20:31 -0700 Subject: [PATCH 099/152] Bump circleci Python from 3.10 -> 3.13. (#264) --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 716a2610..5e482ebe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: build-docs: working_directory: ~/repo docker: - - image: cimg/python:3.10 + - image: cimg/python:3.13 steps: - checkout @@ -49,7 +49,7 @@ jobs: deploy-docs: working_directory: ~/repo docker: - - image: cimg/python:3.10 + - image: cimg/python:3.13 steps: - checkout From 48c522ca97421c90d945a48565ea0c3745333d4b Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Tue, 23 Sep 2025 13:27:54 -0700 Subject: [PATCH 100/152] MAINT: Reduce timeout config value for CI safety. (#265) --- site/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/conf.py b/site/conf.py index 4aea25ca..6b80e054 100644 --- a/site/conf.py +++ b/site/conf.py @@ -47,7 +47,7 @@ ] # MyST-NB configuration -nb_execution_timeout = 900 +nb_execution_timeout = 60 # -- Options for HTML output ------------------------------------------------- From bb96c4ab8f02030cf3d80b9e0dff63d34be3ce68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 29 Sep 2025 22:31:41 -0700 Subject: [PATCH 101/152] CI: Update testing matrices and minimum supported versions (#266) * CI: adding python 3.13 and 3.14 to the tests * CI: removing support for py3.9 and bumping minimum versions of dependencies * CI: fixing versions in ci config --- .github/workflows/ci_tests_run_notebooks.yml | 18 +++++++++--------- tox.ini | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index dfd48f58..f76e7fba 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -20,14 +20,14 @@ jobs: matrix: # Run all supported OS for one Python version, then add a few extra scenarios os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.10'] - toxenv: [py310-test] - name: ['with Python 3.10',] + python-version: ['3.13'] + toxenv: [py313-test] + name: ['with Python 3.13',] include: - - python-version: '3.9' - toxenv: py39-test-oldestdeps - name: with Python 3.9 and oldest versioned dependencies + - python-version: '3.10' + toxenv: py310-test-oldestdeps + name: with Python 3.10 and oldest versioned dependencies os: ubuntu-latest - python-version: '3.11' @@ -40,9 +40,9 @@ jobs: name: with Python 3.12 and latest or pre-release version of dependencies os: ubuntu-latest - - python-version: '3.12' - toxenv: py312-test-devdeps - name: with Python 3.12 and developer versioned dependencies + - python-version: '3.14-dev' + toxenv: py314-test-devdeps + name: with Python 3.14 and developer versioned dependencies os: ubuntu-latest steps: diff --git a/tox.ini b/tox.ini index ff48327d..f1198620 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{39,310,311,312}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} + py{310,311,312,313,314}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} requires = pip >= 19.3.1 @@ -18,10 +18,10 @@ deps = -rsite/requirements.txt -rrequirements.txt - oldestdeps: numpy==1.23 - oldestdeps: matplotlib==3.6 - oldestdeps: scipy==1.8 - oldestdeps: pandas==1.4 + oldestdeps: numpy==1.24 + oldestdeps: matplotlib==3.7 + oldestdeps: scipy==1.10 + oldestdeps: pandas==2.0 devdeps: numpy>=0.0.dev0 devdeps: scipy>=0.0.dev0 From b1e0ccae70593f9e9245e472d8a1013144ecbbcd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Oct 2025 12:07:31 +0000 Subject: [PATCH 102/152] Bump the actions group in /.github/workflows with 2 updates Bumps the actions group in /.github/workflows with 2 updates: [actions/setup-python](https://github.com/actions/setup-python) and [scientific-python/circleci-artifacts-redirector-action](https://github.com/scientific-python/circleci-artifacts-redirector-action). Updates `actions/setup-python` from 5.6.0 to 6.0.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/a26af69be951a213d495a4c3e4e4022e16d87065...e797f83bcb11b83ae66e0230d6156d7c80228e7c) Updates `scientific-python/circleci-artifacts-redirector-action` from 1.2.0 to 1.3.1 - [Release notes](https://github.com/scientific-python/circleci-artifacts-redirector-action/releases) - [Commits](https://github.com/scientific-python/circleci-artifacts-redirector-action/compare/839631420e45a08af893032e5a5e8843bf47e8ff...5d358ff96e96429a5c64a969bb4a574555439f4f) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: scientific-python/circleci-artifacts-redirector-action dependency-version: 1.3.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci_tests_run_notebooks.yml | 2 +- .github/workflows/circleci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml index f76e7fba..5bd70740 100644 --- a/.github/workflows/ci_tests_run_notebooks.yml +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -48,7 +48,7 @@ jobs: steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 #v5.6.0 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c #v6.0.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index 642252b7..35c4c076 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -5,7 +5,7 @@ jobs: name: Run CircleCI artifacts redirector steps: - name: GitHub Action step - uses: scientific-python/circleci-artifacts-redirector-action@839631420e45a08af893032e5a5e8843bf47e8ff # v1.2.0 + uses: scientific-python/circleci-artifacts-redirector-action@5d358ff96e96429a5c64a969bb4a574555439f4f # v1.3.1 with: repo-token: ${{ secrets.GITHUB_TOKEN }} api-token: ${{ secrets.CIRCLE_TOKEN }} From ca0f62bda7572b18bc957953ead685a682a7df5c Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Fri, 3 Oct 2025 21:44:03 -0700 Subject: [PATCH 103/152] DOC: Update the URL to a backpropagation LSTM article --- content/tutorial-nlp-from-scratch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index a4771883..67b3dce5 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -652,7 +652,7 @@ def initialize_grads(parameters): return grads ``` -Now, for each gate and the fully connected layer, we define a function to calculate the gradient of the loss with respect to the input passed and the parameters used. To understand the mathematics behind how the derivatives were calculated we suggest you to follow this helpful [blog](https://christinakouridi.blog/2019/06/19/backpropagation-lstm/) by Christina Kouridi. +Now, for each gate and the fully connected layer, we define a function to calculate the gradient of the loss with respect to the input passed and the parameters used. To understand the mathematics behind how the derivatives were calculated we suggest you to follow this helpful [blog](https://christinakouridi.github.io/posts/backprop-lstm/) by Christina Kouridi. Define a function to calculate the gradients in the **Forget Gate**: From b28ffe4a4bbfc253f73c0ab9548b9859130a6206 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Nov 2025 12:05:34 +0000 Subject: [PATCH 104/152] Bump actions/upload-artifact in /.github/workflows in the actions group Bumps the actions group in /.github/workflows with 1 update: [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/upload-artifact` from 4.6.2 to 5.0.0 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/ea165f8d65b6e75b540449e92b4886f43607fa02...330a01c490aca151604b8cf639adc76d48f6c5d4) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: 5.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 5a3a9039..0d2bc0f0 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -42,7 +42,7 @@ jobs: conda list make -C site/ SPHINXOPTS="-nWT --keep-going" html - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: sphinx-build-artifact path: site/_build/html/reports From 153a2a744920d6131be382799a09d9a47cfd6e53 Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Thu, 6 Nov 2025 05:40:40 -0800 Subject: [PATCH 105/152] DOC: Correct typos in numpy tutorials (#269) * DOC: traini -> train * DOC: Correct typos in mooreslaw-tutorial.md * DOC: Correct punctuation usage in Sentiment Analysis tutorial * DOC: weights values -> weights * Update content/mooreslaw-tutorial.md Co-authored-by: Ross Barnowski --- content/mooreslaw-tutorial.md | 10 ++--- content/tutorial-deep-learning-on-mnist.md | 2 +- ...orcement-learning-with-pong-from-pixels.md | 2 +- content/tutorial-nlp-from-scratch.md | 38 ++++++++++--------- 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/content/mooreslaw-tutorial.md b/content/mooreslaw-tutorial.md index 42c0de15..cedc32d9 100644 --- a/content/mooreslaw-tutorial.md +++ b/content/mooreslaw-tutorial.md @@ -106,7 +106,7 @@ $B_M=-675.4$ Since the function represents Moore's law, define it as a Python function using -[`lambda`](https://docs.python.org/3/library/ast.html?highlight=lambda#ast.Lambda) +[`lambda`](https://docs.python.org/3/library/ast.html?highlight=lambda#ast.Lambda): ```{code-cell} A_M = np.log(2) / 2 @@ -156,7 +156,7 @@ The extra options below will put the data in the desired format: * `delimiter = ','`: specify delimeter as a comma ',' (this is the default behavior) * `usecols = [1,2]`: import the second and third columns from the csv -* `skiprows = 1`: do not use the first row, because its a header row +* `skiprows = 1`: do not use the first row, because it's a header row ```{code-cell} data = np.loadtxt("transistor_data.csv", delimiter=",", usecols=[1, 2], skiprows=1) @@ -282,7 +282,7 @@ In the next plot, use the [`fivethirtyeight`](https://matplotlib.org/3.1.1/gallery/style_sheets/fivethirtyeight.html) style sheet. The style sheet replicates -https://fivethirtyeight.com elements. Change the matplotlib style with + elements. Change the matplotlib style with [`plt.style.use`](https://matplotlib.org/3.3.2/api/style_api.html#matplotlib.style.use). ```{code-cell} @@ -334,7 +334,7 @@ option, to increase the transparency of the data. The more opaque the points appear, the more reported values lie on that measurement. The green $+$ is the average reported transistor count for 2017. Plot your predictions -for $\pm\frac{1}{2}~years. +for $\pm\frac{1}{2}$ years. ```{code-cell} transistor_count2017 = transistor_count[year == 2017] @@ -386,7 +386,7 @@ array using `np.loadtxt`, to save your model use two approaches ### Zipping the arrays into a file Using `np.savez`, you can save thousands of arrays and give them names. The function `np.load` will load the arrays back into the workspace as a -dictionary. You'll save a five arrays so the next user will have the year, +dictionary. You'll save five arrays so the next user will have the year, transistor count, predicted transistor count, Gordon Moore's predicted count, and fitting constants. Add one more variable that other users can use to understand the model, `notes`. diff --git a/content/tutorial-deep-learning-on-mnist.md b/content/tutorial-deep-learning-on-mnist.md index e1085b91..d58f4f7b 100644 --- a/content/tutorial-deep-learning-on-mnist.md +++ b/content/tutorial-deep-learning-on-mnist.md @@ -384,7 +384,7 @@ layer.)](_static/tutorial-deep-learning-on-mnist.png) In the beginning of model training, your network randomly initializes the weights and feeds the input data forward through the hidden and output layers. This process is the forward pass or forward propagation. - Then, the network propagates the "signal" from the loss function back through the hidden layer and adjusts the weights values with the help of the learning rate parameter (more on that later). + Then, the network propagates the "signal" from the loss function back through the hidden layer and adjusts the weights with the help of the learning rate parameter (more on that later). > **Note:** In more technical terms, you: > diff --git a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md index 1598e572..12a789ef 100644 --- a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md +++ b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md @@ -552,7 +552,7 @@ while episode_number < max_episodes: A few notes: -- If you have previously run an experiment and want to repeat it, your `Monitor` instance may still be running, which may throw an error the next time you try to traini the agent. Therefore, you should first shut down `Monitor` by calling `env.close()` by uncommenting and running the cell below: +- If you have previously run an experiment and want to repeat it, your `Monitor` instance may still be running, which may throw an error the next time you try to train the agent. Therefore, you should first shut down `Monitor` by calling `env.close()` by uncommenting and running the cell below: ```python # env.close() diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index 67b3dce5..49a84ed5 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -104,7 +104,7 @@ We made sure to include different demographics in our data and included a range ## 2. Preprocess the datasets >Preprocessing data is an extremely crucial step before building any Deep learning model, however in an attempt to keep the tutorial focused on building the model, we will not dive deep into the code for preprocessing. Given below is a brief overview of all the steps we undertake to clean our data and convert it to its numeric representation. -1. **Text Denoising** : Before converting your text into vectors, it is important to clean it and remove all unhelpful parts a.k.a the noise from your data by converting all characters to lowercase, removing html tags, brackets and stop words (words that don't add much meaning to a sentence). Without this step the dataset is often a cluster of words that the computer doesn't understand. +1. **Text Denoising** : Before converting your text into vectors, it is important to clean it and remove all unhelpful parts a.k.a. the noise from your data by converting all characters to lowercase, removing html tags, brackets and stop words (words that don't add much meaning to a sentence). Without this step the dataset is often a cluster of words that the computer doesn't understand. 2. **Converting words to vectors** : A word embedding is a learned representation for text where words that have the same meaning have a similar representation. Individual words are represented as real-valued vectors in a predefined vector space. GloVe is an unsupervised algorithm developed by Stanford for generating word embeddings by generating global word-word co-occurence matrix from a corpus. You can download the zipped files containing the embeddings from [the GloVe official website](https://nlp.stanford.edu/projects/glove/). Here you can choose any of the four options for different sizes or training datasets. We have chosen the least memory consuming embedding file. @@ -390,7 +390,7 @@ imdb_train = data.fetch('imdb_train.txt') imdb_test = data.fetch('imdb_test.txt') ``` -Instantiate the` TextPreprocess` class to perform various operations on our datasets: +Instantiate the `TextPreprocess` class to perform various operations on our datasets: ```python textproc = TextPreprocess() @@ -421,7 +421,7 @@ y_test = test_df['sentiment'].to_numpy()[0:1000] ``` The same process is applicable on the collected speeches: -> Since we will be performing paragraph wise sentiment analysis on each speech further ahead in the tutorial, we'll need the punctuation marks to split the text into paragraphs, hence we refrain from removing their punctuation marks at this stage +> Since we will be performing paragraph wise sentiment analysis on each speech further ahead in the tutorial, we'll need the punctuation marks to split the text into paragraphs, hence we refrain from removing their punctuation marks at this stage. ```python speech_data_path = 'tutorial-nlp-from-scratch/speeches.csv' @@ -444,13 +444,13 @@ emb_matrix = textproc.loadGloveModel(emb_path) ## 3. Build the Deep Learning Model It is time to start implementing our LSTM! You will have to first familiarize yourself with some high-level concepts of the basic building blocks of a deep learning model. You can refer to the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) for the same. -You will then learn how a Recurrent Neural Network differs from a plain Neural Network and what makes it so suitable for processing sequential data. Afterwards, you will construct the building blocks of a simple deep learning model in Python and NumPy and train it to learn to classify the sentiment of a piece of text as positive or negative with a certain level of accuracy +You will then learn how a Recurrent Neural Network differs from a plain Neural Network and what makes it so suitable for processing sequential data. Afterwards, you will construct the building blocks of a simple deep learning model in Python and NumPy and train it to learn to classify the sentiment of a piece of text as positive or negative with a certain level of accuracy. ### Introduction to a Long Short Term Memory Network In a [Multilayer perceptron](https://en.wikipedia.org/wiki/Multilayer_perceptron) (MLP), the information only moves in one direction — from the input layer, through the hidden layers, to the output layer. The information moves straight through the network and never takes the previous nodes into account at a later stage. Because it only considers the current input, the features learned are not shared across different positions of the sequence. Moreover, it cannot process sequences with varying lengths. -Unlike an MLP, the RNN was designed to work with sequence prediction problems.RNNs introduce state variables to store past information, together with the current inputs, to determine the current outputs. Since an RNN shares the learned features with all the data points in a sequence regardless of its length, it is capable of processing sequences with varying lengths. +Unlike an MLP, the RNN was designed to work with sequence prediction problems. RNNs introduce state variables to store past information, together with the current inputs, to determine the current outputs. Since an RNN shares the learned features with all the data points in a sequence regardless of its length, it is capable of processing sequences with varying lengths. The problem with an RNN however, is that it cannot retain long-term memory because the influence of a given input on the hidden layer, and therefore on the network output, either decays or blows up exponentially as it cycles around the network’s recurrent connections. This shortcoming is referred to as the vanishing gradient problem. Long Short-Term Memory (LSTM) is an RNN architecture specifically designed to address the [vanishing gradient problem](https://en.wikipedia.org/wiki/Vanishing_gradient_problem). @@ -462,7 +462,7 @@ The problem with an RNN however, is that it cannot retain long-term memory becau In the above gif, the rectangles labeled $A$ are called `Cells` and they are the **Memory Blocks** of our LSTM network. They are responsible for choosing what to remember in a sequence and pass on that information to the next cell via two states called the `hidden state` $H_{t}$ and the `cell state` $C_{t}$ where $t$ indicates the time-step. Each `Cell` has dedicated gates which are responsible for storing, writing or reading the information passed to an LSTM. You will now look closely at the architecture of the network by implementing each mechanism happening inside of it. -Lets start with writing a function to randomly initialize the parameters which will be learned while our model trains +Lets start with writing a function to randomly initialize the parameters which will be learned while our model trains: ```python def initialise_params(hidden_dim, input_dim): @@ -641,7 +641,7 @@ def forward_prop(X_vec, parameters, input_dim): After each forward pass through the network, you will implement the `backpropagation through time` algorithm to accumulate gradients of each parameter over the time steps. Backpropagation through a LSTM is not as straightforward as through other common Deep Learning architectures, due to the special way its underlying layers interact. Nonetheless, the approach is largely the same; identifying dependencies and applying the chain rule. -Lets start with defining a function to initialize gradients of each parameter as arrays made up of zeros with same dimensions as the corresponding parameter +Lets start with defining a function to initialize gradients of each parameter as arrays made up of zeros with same dimensions as the corresponding parameter. ```python # Initialise the gradients @@ -777,10 +777,10 @@ def backprop(y, caches, hidden_dim, input_dim, time_steps, parameters): ### Updating the Parameters -We update the parameters through an optimization algorithm called [Adam](https://optimization.cbe.cornell.edu/index.php?title=Adam) which is an extension to stochastic gradient descent that has recently seen broader adoption for deep learning applications in computer vision and natural language processing. Specifically, the algorithm calculates an exponential moving average of the gradient and the squared gradient, and the parameters `beta1` and `beta2` control the decay rates of these moving averages. Adam has shown increased convergence and robustness over other gradient descent algorithms and is often recommended as the default optimizer for training. +We update the parameters through an optimization algorithm called [Adam](https://optimization.cbe.cornell.edu/index.php?title=Adam), which is an extension to stochastic gradient descent that has recently seen broader adoption for deep learning applications in computer vision and natural language processing. Specifically, the algorithm calculates an exponential moving average of the gradient and the squared gradient, and the parameters `beta1` and `beta2` control the decay rates of these moving averages. Adam has shown increased convergence and robustness over other gradient descent algorithms, and is often recommended as the default optimizer for training. -Define a function to initialise the moving averages for each parameter +Define a function to initialise the moving averages for each parameter: ```python # initialise the moving averages @@ -795,7 +795,7 @@ def initialise_mav(hidden_dim, input_dim, params): return v, s ``` -Define a function to update the parameters +Define a function to update the parameters: ```python # Update the parameters using Adam optimization @@ -820,7 +820,7 @@ def update_parameters(parameters, gradients, v, s, ### Training the Network -You will start by initializing all the parameters and hyperparameters being used in your network +You will start by initializing all the parameters and hyperparameters being used in your network: ```python hidden_dim = 64 @@ -834,8 +834,10 @@ v, s = initialise_mav(hidden_dim, parameters) ``` -To optimize your deep learning network, you need to calculate a loss based on how well the model is doing on the training data. Loss value implies how poorly or well a model behaves after each iteration of optimization.
-Define a function to calculate the loss using [negative log likelihood](http://d2l.ai/chapter_linear-networks/softmax-regression.html?highlight=negative%20log%20likelihood#log-likelihood) +To optimize your deep learning network, you need to calculate a loss based on how well the model is doing on the training data. Loss value implies how poorly or well a model behaves after each iteration of optimization. + + +Define a function to calculate the loss using [negative log likelihood](http://d2l.ai/chapter_linear-networks/softmax-regression.html?highlight=negative%20log%20likelihood#log-likelihood): ```python def loss_f(A, Y): @@ -849,7 +851,7 @@ def loss_f(A, Y): ``` Set up the neural network's learning experiment with a training loop and start the training process. You will also evaluate the model's performance on the training dataset to see how well the model is *learning* and the testing dataset to see how well it is *generalizing*. ->Skip running this cell if you already have the trained parameters stored in a `npy` file +>Skip running this cell if you already have the trained parameters stored in a `npy` file. ```python # To store training losses @@ -952,7 +954,7 @@ plt.show() ### Sentiment Analysis on the Speech Data -Once your model is trained, you can use the updated parameters to start making our predictions. You can break each speech into paragraphs of uniform size before passing them to the Deep Learning model and predicting the sentiment of each paragraph +Once your model is trained, you can use the updated parameters to start making our predictions. You can break each speech into paragraphs of uniform size before passing them to the Deep Learning model and predicting the sentiment of each paragraph. ```python # To store predicted sentiments @@ -1028,7 +1030,7 @@ In the plot above, you're shown what percentages of each speech are expected to It's crucial to understand that accurately identifying a text's sentiment is not easy primarily because of the complex ways in which humans express sentiment, using irony, sarcasm, humor, or, in social media, abbreviation. Moreover neatly placing text into two categories: 'positive' and 'negative' can be problematic because it is being done without any context. Words or abbreviations can convey very different sentiments depending on age and location, none of which we took into account while building our model. -Along with data, there are also growing concerns that data processing algorithms are influencing policy and daily lives in ways that are not transparent and introduce biases. Certain biases such as the [Inductive Bias](https://bit.ly/2WtTKIe) are essential to help a Machine Learning model generalize better, for example the LSTM we built earlier is biased towards preserving contextual information over long sequences which makes it so suitable for processing sequential data. The problem arises when [societal biases](https://hbr.org/2019/10/what-do-we-do-about-the-biases-in-ai) creep into algorithmic predictions. Optimizing Machine algorithms via methods like [hyperparameter tuning](https://en.wikipedia.org/wiki/Hyperparameter_optimization) can then further amplify these biases by learning every bit of information in the data. +Along with data, there are also growing concerns that data processing algorithms are influencing policy and daily lives in ways that are not transparent and introduce biases. Certain biases such as the [Inductive Bias](https://bit.ly/2WtTKIe) are essential to help a Machine Learning model generalize better, for example the LSTM we built earlier is biased towards preserving contextual information over long sequences which makes it so suitable for processing sequential data. The problem arises when [societal biases](https://hbr.org/2019/10/what-do-we-do-about-the-biases-in-ai) creep into algorithmic predictions. Optimizing Machine Learning algorithms via methods like [hyperparameter tuning](https://en.wikipedia.org/wiki/Hyperparameter_optimization) can then further amplify these biases by learning every bit of information in the data. There are also cases where bias is only in the output and not the inputs (data, algorithm). For example, in sentiment analysis [accuracy tends to be higher on female-authored texts than on male-authored ones]( https://doi.org/10.3390/electronics9020374). End users of sentiment analysis should be aware that its small gender biases can affect the conclusions drawn from it and apply correction factors when necessary. Hence, it is important that demands for algorithmic accountability should include the ability to test the outputs of a system, including the ability to drill down into different user groups by gender, ethnicity and other characteristics, to identify, and hopefully suggest corrections for, system output biases. @@ -1039,7 +1041,7 @@ There are also cases where bias is only in the output and not the inputs (data, You have learned how to build and train a simple Long Short Term Memory network from scratch using just NumPy to perform sentiment analysis. -To further enhance and optimize your neural network model, you can consider one of a mixture of the following: +To further enhance and optimize your neural network model, you can consider one or a mixture of the following: - Alter the architecture by introducing multiple LSTM layers to make the network deeper. - Use a higher epoch size to train longer and add more regularization techniques, such as early stopping, to prevent overfitting. @@ -1053,7 +1055,7 @@ Nowadays, LSTMs have been replaced by the [Transformer](https://jalammar.github. Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as PyTorch, JAX or TensorFlow — that provide NumPy-like APIs, have built-in automatic differentiation and GPU support, and are designed for high-performance numerical computing and machine learning. -Finally, to know more about how ethics come into play when developing a machine learning model, you can refer to the following resources : +Finally, to know more about how ethics come into play when developing a machine learning model, you can refer to the following resources: - [Data ethics resources](https://www.turing.ac.uk/research/data-ethics) by the Turing Institute - Considering how artificial intelligence shifts power, an [article](https://www.nature.com/articles/d41586-020-02003-2) and [talk](https://slideslive.com/38923453/the-values-of-machine-learning) by Pratyusha Kalluri - More ethics resources on [this blog post](https://www.fast.ai/2018/09/24/ai-ethics-resources/) by Rachel Thomas and the [Radical AI podcast](https://www.radicalai.org/) From 32acc627b23e3e7a2ae90f484c0bc486d8f23bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 07:48:46 -0800 Subject: [PATCH 106/152] CI: switching circleCI to use new engine and use proxy for artifacts (#271) * CI: switching circleCI to use new engine and use proxy for artifacts * Renaming workflow to be consistent with other repos * Adding GHA deployment for the rendered pages --- .circleci/config.yml | 81 +++---------------- .github/workflows/ci_publish.yml | 46 +++++++++++ ....yml => circleci-artifacts-redirector.yml} | 2 + 3 files changed, 60 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/ci_publish.yml rename .github/workflows/{circleci.yml => circleci-artifacts-redirector.yml} (83%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e482ebe..c701888d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,91 +1,34 @@ -# See: https://circleci.com/docs/2.0/language-python/ +version: 2.1 -version: 2 -jobs: +jobs: build-docs: - working_directory: ~/repo docker: - image: cimg/python:3.13 - steps: - checkout - run: name: Install Python dependencies command: | - python3 -m venv venv - source venv/bin/activate - pip install --upgrade pip wheel setuptools - pip install -r site/requirements.txt -r requirements.txt - - - restore_cache: - keys: - - cache-data + python -m pip install --upgrade pip tox - run: - name: Build site - no_output_timeout: 30m + name: Build documentation + no_output_timeout: 60m + environment: + # Ensure this is same as store_artifacts path below + DOCS_PATH: _build/html command: | - # NOTE: blas multithreading behaves badly on circleci + export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH" export OMP_NUM_THREADS=1 - source venv/bin/activate - # n = nitpicky (broken links), W = warnings as errors, - # T = full tracebacks, keep-going = run to completion even with errors - make -C site/ SPHINXOPTS="-nWT --keep-going" html - - - save_cache: - key: cache-data - paths: - - _data + python -m tox -e py313-buildhtml - store_artifacts: - path: site/_build/html - - - persist_to_workspace: - root: site/_build - paths: html - - deploy-docs: - working_directory: ~/repo - docker: - - image: cimg/python:3.13 - steps: - - checkout - - - attach_workspace: - at: site/_build - - - run: - name: install deploy deps - command : | - python3 -m pip install --user ghp-import - - - run: - name: configure git - command: | - git config --global user.name "ci-doc-deploy-bot" - git config --global user.email "ci-doc-deploy-bot@nomail" - git config --global push.default simple - - - add_ssh_keys: - fingerprints: - 5c:54:62:37:75:7f:4d:14:f4:07:82:1c:50:0d:ee:9b - - - run: - name: deploy to gh-pages - command: | - ghp-import -n -f -p -m "[skip ci] docs build of $CIRCLE_SHA1" site/_build/html - + path: _build/html workflows: version: 2 - build: + build-and-docs: jobs: - build-docs - - deploy-docs: - requires: - - build-docs - filters: - branches: - only: main diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml new file mode 100644 index 00000000..510284af --- /dev/null +++ b/.github/workflows/ci_publish.yml @@ -0,0 +1,46 @@ +name: Build and Publish HTML and deployed_notebooks + +on: + push: + branches: + - main + schedule: + - cron: '0 5 * * 1' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # `BASE_URL` determines, relative to the root of the domain, the URL that your site is served from. + # E.g., if your site lives at `https://mydomain.org/myproject`, set `BASE_URL=/myproject`. + # If, instead, your site lives at the root of the domain, at `https://mydomain.org`, set `BASE_URL=''`. + BASE_URL: /${{ github.event.repository.name }} + +jobs: + + publish_html: + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + name: Publish HTML + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install dependencies + run: python -m pip install --upgrade tox + + - name: Execute notebooks while building HTMLs + run: tox -e py312-buildhtml + + - name: Publish + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./_build/html/ + commit_message: ${{ github.event.head_commit.message }} diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci-artifacts-redirector.yml similarity index 83% rename from .github/workflows/circleci.yml rename to .github/workflows/circleci-artifacts-redirector.yml index 35c4c076..95a7e6b8 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci-artifacts-redirector.yml @@ -11,3 +11,5 @@ jobs: api-token: ${{ secrets.CIRCLE_TOKEN }} artifact-path: 0/site/_build/html/index.html circleci-jobs: build-docs + domain: circle.scientific-python.dev + job-title: "--> Rendering Preview <--" From 6499d1f0bdc7a53769ea1ce101ea2aefe59f2b5f Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 6 Nov 2025 08:05:10 -0800 Subject: [PATCH 107/152] =?UTF-8?q?Revert=20"CI:=20switching=20circleCI=20?= =?UTF-8?q?to=20use=20new=20engine=20and=20use=20proxy=20for=20artifacts?= =?UTF-8?q?=20=E2=80=A6"=20(#272)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 32acc627b23e3e7a2ae90f484c0bc486d8f23bd0. --- .circleci/config.yml | 81 ++++++++++++++++--- .github/workflows/ci_publish.yml | 46 ----------- ...-artifacts-redirector.yml => circleci.yml} | 2 - 3 files changed, 69 insertions(+), 60 deletions(-) delete mode 100644 .github/workflows/ci_publish.yml rename .github/workflows/{circleci-artifacts-redirector.yml => circleci.yml} (83%) diff --git a/.circleci/config.yml b/.circleci/config.yml index c701888d..5e482ebe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,34 +1,91 @@ -version: 2.1 - +# See: https://circleci.com/docs/2.0/language-python/ +version: 2 jobs: + build-docs: + working_directory: ~/repo docker: - image: cimg/python:3.13 + steps: - checkout - run: name: Install Python dependencies command: | - python -m pip install --upgrade pip tox + python3 -m venv venv + source venv/bin/activate + pip install --upgrade pip wheel setuptools + pip install -r site/requirements.txt -r requirements.txt + + - restore_cache: + keys: + - cache-data - run: - name: Build documentation - no_output_timeout: 60m - environment: - # Ensure this is same as store_artifacts path below - DOCS_PATH: _build/html + name: Build site + no_output_timeout: 30m command: | - export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH" + # NOTE: blas multithreading behaves badly on circleci export OMP_NUM_THREADS=1 - python -m tox -e py313-buildhtml + source venv/bin/activate + # n = nitpicky (broken links), W = warnings as errors, + # T = full tracebacks, keep-going = run to completion even with errors + make -C site/ SPHINXOPTS="-nWT --keep-going" html + + - save_cache: + key: cache-data + paths: + - _data - store_artifacts: - path: _build/html + path: site/_build/html + + - persist_to_workspace: + root: site/_build + paths: html + + deploy-docs: + working_directory: ~/repo + docker: + - image: cimg/python:3.13 + steps: + - checkout + + - attach_workspace: + at: site/_build + + - run: + name: install deploy deps + command : | + python3 -m pip install --user ghp-import + + - run: + name: configure git + command: | + git config --global user.name "ci-doc-deploy-bot" + git config --global user.email "ci-doc-deploy-bot@nomail" + git config --global push.default simple + + - add_ssh_keys: + fingerprints: + 5c:54:62:37:75:7f:4d:14:f4:07:82:1c:50:0d:ee:9b + + - run: + name: deploy to gh-pages + command: | + ghp-import -n -f -p -m "[skip ci] docs build of $CIRCLE_SHA1" site/_build/html + workflows: version: 2 - build-and-docs: + build: jobs: - build-docs + - deploy-docs: + requires: + - build-docs + filters: + branches: + only: main diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml deleted file mode 100644 index 510284af..00000000 --- a/.github/workflows/ci_publish.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Build and Publish HTML and deployed_notebooks - -on: - push: - branches: - - main - schedule: - - cron: '0 5 * * 1' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # `BASE_URL` determines, relative to the root of the domain, the URL that your site is served from. - # E.g., if your site lives at `https://mydomain.org/myproject`, set `BASE_URL=/myproject`. - # If, instead, your site lives at the root of the domain, at `https://mydomain.org`, set `BASE_URL=''`. - BASE_URL: /${{ github.event.repository.name }} - -jobs: - - publish_html: - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Publish HTML - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - - - name: Install dependencies - run: python -m pip install --upgrade tox - - - name: Execute notebooks while building HTMLs - run: tox -e py312-buildhtml - - - name: Publish - uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./_build/html/ - commit_message: ${{ github.event.head_commit.message }} diff --git a/.github/workflows/circleci-artifacts-redirector.yml b/.github/workflows/circleci.yml similarity index 83% rename from .github/workflows/circleci-artifacts-redirector.yml rename to .github/workflows/circleci.yml index 95a7e6b8..35c4c076 100644 --- a/.github/workflows/circleci-artifacts-redirector.yml +++ b/.github/workflows/circleci.yml @@ -11,5 +11,3 @@ jobs: api-token: ${{ secrets.CIRCLE_TOKEN }} artifact-path: 0/site/_build/html/index.html circleci-jobs: build-docs - domain: circle.scientific-python.dev - job-title: "--> Rendering Preview <--" From 1b2473d2755850b2442b23dcb950ae139361fdfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 08:22:42 -0800 Subject: [PATCH 108/152] CI: switching circleCI to use new engine and use proxy for artifacts (#273) * CI: switching circleCI to use new engine and use proxy for artifacts * Renaming workflow to be consistent with other repos --- .circleci/config.yml | 81 +++---------------- ....yml => circleci-artifacts-redirector.yml} | 2 + 2 files changed, 14 insertions(+), 69 deletions(-) rename .github/workflows/{circleci.yml => circleci-artifacts-redirector.yml} (83%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e482ebe..c701888d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,91 +1,34 @@ -# See: https://circleci.com/docs/2.0/language-python/ +version: 2.1 -version: 2 -jobs: +jobs: build-docs: - working_directory: ~/repo docker: - image: cimg/python:3.13 - steps: - checkout - run: name: Install Python dependencies command: | - python3 -m venv venv - source venv/bin/activate - pip install --upgrade pip wheel setuptools - pip install -r site/requirements.txt -r requirements.txt - - - restore_cache: - keys: - - cache-data + python -m pip install --upgrade pip tox - run: - name: Build site - no_output_timeout: 30m + name: Build documentation + no_output_timeout: 60m + environment: + # Ensure this is same as store_artifacts path below + DOCS_PATH: _build/html command: | - # NOTE: blas multithreading behaves badly on circleci + export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH" export OMP_NUM_THREADS=1 - source venv/bin/activate - # n = nitpicky (broken links), W = warnings as errors, - # T = full tracebacks, keep-going = run to completion even with errors - make -C site/ SPHINXOPTS="-nWT --keep-going" html - - - save_cache: - key: cache-data - paths: - - _data + python -m tox -e py313-buildhtml - store_artifacts: - path: site/_build/html - - - persist_to_workspace: - root: site/_build - paths: html - - deploy-docs: - working_directory: ~/repo - docker: - - image: cimg/python:3.13 - steps: - - checkout - - - attach_workspace: - at: site/_build - - - run: - name: install deploy deps - command : | - python3 -m pip install --user ghp-import - - - run: - name: configure git - command: | - git config --global user.name "ci-doc-deploy-bot" - git config --global user.email "ci-doc-deploy-bot@nomail" - git config --global push.default simple - - - add_ssh_keys: - fingerprints: - 5c:54:62:37:75:7f:4d:14:f4:07:82:1c:50:0d:ee:9b - - - run: - name: deploy to gh-pages - command: | - ghp-import -n -f -p -m "[skip ci] docs build of $CIRCLE_SHA1" site/_build/html - + path: _build/html workflows: version: 2 - build: + build-and-docs: jobs: - build-docs - - deploy-docs: - requires: - - build-docs - filters: - branches: - only: main diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci-artifacts-redirector.yml similarity index 83% rename from .github/workflows/circleci.yml rename to .github/workflows/circleci-artifacts-redirector.yml index 35c4c076..95a7e6b8 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci-artifacts-redirector.yml @@ -11,3 +11,5 @@ jobs: api-token: ${{ secrets.CIRCLE_TOKEN }} artifact-path: 0/site/_build/html/index.html circleci-jobs: build-docs + domain: circle.scientific-python.dev + job-title: "--> Rendering Preview <--" From 5dcb7cb7aea5d4497f450f3c7b43ed686f6c64ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 08:35:05 -0800 Subject: [PATCH 109/152] CI: YOLO image name --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c701888d..43f9b4c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ version: 2.1 jobs: build-docs: docker: - - image: cimg/python:3.13 + - image: cimg/python:3.13-node steps: - checkout From 64c925610cc0bc8787e8a52f358e454d67d29690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 19:51:15 -0800 Subject: [PATCH 110/152] CI: fix path in redirect action --- .github/workflows/circleci-artifacts-redirector.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/circleci-artifacts-redirector.yml b/.github/workflows/circleci-artifacts-redirector.yml index 95a7e6b8..f24efe72 100644 --- a/.github/workflows/circleci-artifacts-redirector.yml +++ b/.github/workflows/circleci-artifacts-redirector.yml @@ -1,3 +1,4 @@ +name: Run CircleCI artifacts redirector for rendered HTML on: [status] jobs: circleci_artifacts_redirector_job: @@ -9,7 +10,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} api-token: ${{ secrets.CIRCLE_TOKEN }} - artifact-path: 0/site/_build/html/index.html + artifact-path: 0/_build/html/index.html circleci-jobs: build-docs domain: circle.scientific-python.dev job-title: "--> Rendering Preview <--" From 990c9cfab0ce827aaec48d027db57cefaebf39a7 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 10 Nov 2025 21:09:53 -0800 Subject: [PATCH 111/152] TST: Update tox config to fix timeouts in CI. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index f1198620..34dbfa2e 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,8 @@ description = run tests setenv = devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple +passenv = OMP_NUM_THREADS + deps = # We use these files to specify all the dependencies, and below we override # versions for specific testing schenarios From 7bdf684d4fc5d34b1551557eee89faaed69c59f9 Mon Sep 17 00:00:00 2001 From: "Christine P. Chai" Date: Tue, 11 Nov 2025 09:58:49 -0800 Subject: [PATCH 112/152] DOC: Correct minor grammar issues (#275) * DOC: a LSTM -> an LSTM * DOC: lets start -> let's start * DOC: lets say -> let's say --------- Co-authored-by: Ross Barnowski --- content/tutorial-nlp-from-scratch.md | 6 +++--- content/tutorial-static_equilibrium.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md index 49a84ed5..c5809cdb 100644 --- a/content/tutorial-nlp-from-scratch.md +++ b/content/tutorial-nlp-from-scratch.md @@ -49,7 +49,7 @@ This tutorial can be run locally in an isolated environment, such as [Virtualenv 2. Preprocess the datasets -3. Build and train a LSTM network from scratch +3. Build and train an LSTM network from scratch 4. Perform sentiment analysis on collected speeches @@ -462,7 +462,7 @@ The problem with an RNN however, is that it cannot retain long-term memory becau In the above gif, the rectangles labeled $A$ are called `Cells` and they are the **Memory Blocks** of our LSTM network. They are responsible for choosing what to remember in a sequence and pass on that information to the next cell via two states called the `hidden state` $H_{t}$ and the `cell state` $C_{t}$ where $t$ indicates the time-step. Each `Cell` has dedicated gates which are responsible for storing, writing or reading the information passed to an LSTM. You will now look closely at the architecture of the network by implementing each mechanism happening inside of it. -Lets start with writing a function to randomly initialize the parameters which will be learned while our model trains: +Let's start with writing a function to randomly initialize the parameters which will be learned while our model trains: ```python def initialise_params(hidden_dim, input_dim): @@ -641,7 +641,7 @@ def forward_prop(X_vec, parameters, input_dim): After each forward pass through the network, you will implement the `backpropagation through time` algorithm to accumulate gradients of each parameter over the time steps. Backpropagation through a LSTM is not as straightforward as through other common Deep Learning architectures, due to the special way its underlying layers interact. Nonetheless, the approach is largely the same; identifying dependencies and applying the chain rule. -Lets start with defining a function to initialize gradients of each parameter as arrays made up of zeros with same dimensions as the corresponding parameter. +Let's start with defining a function to initialize gradients of each parameter as arrays made up of zeros with same dimensions as the corresponding parameter. ```python # Initialise the gradients diff --git a/content/tutorial-static_equilibrium.md b/content/tutorial-static_equilibrium.md index f649e56e..868d777f 100644 --- a/content/tutorial-static_equilibrium.md +++ b/content/tutorial-static_equilibrium.md @@ -190,7 +190,7 @@ The pole does not move, so it must apply a reaction force. The pole also does not rotate, so it must also be creating a reaction moment. Solve for both the reaction force and moments. -Lets say a 5N force is applied perpendicularly 2m above the base of the pole. +Let's say a 5N force is applied perpendicularly 2m above the base of the pole. ```{code-cell} f = 5 # Force in newtons From 1682d2b20b489ae1d781ec765b5423d6b3400d66 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sat, 8 Nov 2025 14:20:04 -0800 Subject: [PATCH 113/152] MAINT: rm todo related to scipy minver. --- content/tutorial-svd.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 494a4d3b..50fb1d72 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -38,11 +38,7 @@ After this tutorial, you should be able to: In this tutorial, we will use a [matrix decomposition](https://en.wikipedia.org/wiki/Matrix_decomposition) from linear algebra, the Singular Value Decomposition, to generate a compressed approximation of an image. We'll use the `face` image from the [scipy.datasets](https://docs.scipy.org/doc/scipy/reference/datasets.html) module: ```{code-cell} -# TODO: Rm try-except with scipy 1.10 is the minimum supported version -try: - from scipy.datasets import face -except ImportError: # Data was in scipy.misc prior to scipy v1.10 - from scipy.misc import face +from scipy.datasets import face img = face() ``` From bdf94bf282bd7c0fd62f661b7b13ef8e95a9ade9 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sat, 8 Nov 2025 14:35:32 -0800 Subject: [PATCH 114/152] MAINT: Switch report order of min/max. --- content/tutorial-svd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 494a4d3b..0fa86a0a 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -115,7 +115,7 @@ You can check that the above works by doing some tests; for example, inquiring about maximum and minimum values for this array: ```{code-cell} -img_array.max(), img_array.min() +img_array.min(), img_array.max() ``` or checking the type of data in the array: From e519e60090a37d8e485627b0a23ab960a81b4d8a Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sat, 8 Nov 2025 14:40:18 -0800 Subject: [PATCH 115/152] MAINT: Access linalg from the np namespace. --- content/tutorial-svd.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 0fa86a0a..c619360f 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -142,12 +142,6 @@ It is possible to use methods from linear algebra to approximate an existing set +++ -To proceed, import the linear algebra submodule from NumPy: - -```{code-cell} -from numpy import linalg -``` - In order to extract information from a given matrix, we can use the SVD to obtain 3 arrays which can be multiplied to obtain the original matrix. From the theory of linear algebra, given a matrix $A$, the following product can be computed: $$U \Sigma V^T = A$$ @@ -183,7 +177,7 @@ plt.show() Now, applying the [linalg.svd](https://numpy.org/devdocs/reference/generated/numpy.linalg.svd.html#numpy.linalg.svd) function to this matrix, we obtain the following decomposition: ```{code-cell} -U, s, Vt = linalg.svd(img_gray) +U, s, Vt = np.linalg.svd(img_gray) ``` **Note** If you are using your own image, this command might take a while to run, depending on the size of your image and your hardware. Don't worry, this is normal! The SVD can be a pretty intensive computation. @@ -221,7 +215,7 @@ Now, we want to check if the reconstructed `U @ Sigma @ Vt` is close to the orig The [linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg) module includes a `norm` function, which computes the norm of a vector or matrix represented in a NumPy array. For example, from the SVD explanation above, we would expect the norm of the difference between `img_gray` and the reconstructed SVD product to be small. As expected, you should see something like ```{code-cell} -linalg.norm(img_gray - U @ Sigma @ Vt) +np.linalg.norm(img_gray - U @ Sigma @ Vt) ``` (The actual result of this operation might be different depending on your architecture and linear algebra setup. Regardless, you should see a small number.) @@ -295,7 +289,7 @@ img_array_transposed.shape Now we are ready to apply the SVD: ```{code-cell} -U, s, Vt = linalg.svd(img_array_transposed) +U, s, Vt = np.linalg.svd(img_array_transposed) ``` Finally, to obtain the full approximated image, we need to reassemble these matrices into the approximation. Now, note that From 990e1c18092d33ccdcfeedca4ffb407fb880e44d Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Sat, 8 Nov 2025 15:03:36 -0800 Subject: [PATCH 116/152] Drop some text in favor of a code comment. --- content/tutorial-svd.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index c619360f..0457e205 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -274,14 +274,10 @@ img_array.shape ``` so we need to permutate the axis on this array to get a shape like `(3, 768, 1024)`. Fortunately, the [numpy.transpose](https://numpy.org/devdocs/reference/generated/numpy.transpose.html#numpy.transpose) function can do that for us: -``` -np.transpose(x, axes=(i, j, k)) -``` -indicates that the axis will be reordered such that the final shape of the transposed array will be reordered according to the indices `(i, j, k)`. - -Let's see how this goes for our array: ```{code-cell} +# The values in the tuple indicate the original dim, and the order the new axis +# so axis 2 -> 0, 0 -> 1, and 1 -> 2 img_array_transposed = np.transpose(img_array, (2, 0, 1)) img_array_transposed.shape ``` From 7f4b387aeee114f72ebf269826ba664979660824 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Wed, 12 Nov 2025 11:17:00 -0800 Subject: [PATCH 117/152] MAINT: Add missing numpy import. --- content/tutorial-svd.md | 1 + 1 file changed, 1 insertion(+) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 0457e205..dd60d758 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -177,6 +177,7 @@ plt.show() Now, applying the [linalg.svd](https://numpy.org/devdocs/reference/generated/numpy.linalg.svd.html#numpy.linalg.svd) function to this matrix, we obtain the following decomposition: ```{code-cell} +import numpy as np U, s, Vt = np.linalg.svd(img_gray) ``` From 7a5ed80ac4a378c6f8586f22caab7621736b9133 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Wed, 12 Nov 2025 13:43:45 -0800 Subject: [PATCH 118/152] Rm duplicated import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Brigitta Sipőcz --- content/tutorial-svd.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index dd60d758..7ccce60a 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -201,8 +201,6 @@ s @ Vt results in a `ValueError`. This happens because having a one-dimensional array for `s`, in this case, is much more economic in practice than building a diagonal matrix with the same data. To reconstruct the original matrix, we can rebuild the diagonal matrix $\Sigma$ with the elements of `s` in its diagonal and with the appropriate dimensions for multiplying: in our case, $\Sigma$ should be 768x1024 since `U` is 768x768 and `Vt` is 1024x1024. In order to add the singular values to the diagonal of `Sigma`, we will use the [fill_diagonal](https://numpy.org/devdocs/reference/generated/numpy.fill_diagonal.html) function from NumPy: ```{code-cell} -import numpy as np - Sigma = np.zeros((U.shape[1], Vt.shape[0])) np.fill_diagonal(Sigma, s) ``` From 6fcce17ca72904b3df3583e6f112a2feb279efcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 06:44:29 -0800 Subject: [PATCH 119/152] ENH: adding myst config and toc --- myst.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 myst.yml diff --git a/myst.yml b/myst.yml new file mode 100644 index 00000000..fa935f15 --- /dev/null +++ b/myst.yml @@ -0,0 +1,35 @@ +version: 1 +project: + title: Numpy Tutorials + # description: + # keywords: [] + authors: Numpy Community + github: https://github.com/numpy/numpy-tutorials + toc: + - file: site/index.md + + - title: Applications + children: + - file: content/mooreslaw-tutorial.md + - file: content/tutorial-deep-learning-on-mnist.md + - file: content/tutorial-x-ray-image-processing.md + - file: content/tutorial-static_equilibrium.md + - file: content/tutorial-plotting-fractals.md + - file: content/tutorial-air-quality-analysis.md + + - title: Features + children: + - file: content/tutorial-svd.md + - file: content/save-load-arrays.md + - file: content/tutorial-ma.md + - title: Contributing + file: site/contributing.md + children: + - file: content/tutorial-style-guide.md + + +site: + template: book-theme + options: + favicon: site/_static/favicon.png + logo: site/_static/numpylogo.svg From d8fca0a5c73a7ca6f73a4ae242c25f2d9995d5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 06:45:17 -0800 Subject: [PATCH 120/152] Removing toctrees and unused index files --- site/applications.md | 17 ----------------- site/articles.md | 13 ------------- site/contributing.md | 15 +-------------- site/features.md | 13 ------------- site/index.md | 20 -------------------- 5 files changed, 1 insertion(+), 77 deletions(-) delete mode 100644 site/applications.md delete mode 100644 site/articles.md delete mode 100644 site/features.md diff --git a/site/applications.md b/site/applications.md deleted file mode 100644 index fa81374e..00000000 --- a/site/applications.md +++ /dev/null @@ -1,17 +0,0 @@ -# NumPy Applications - -A collection of highlighting the use of NumPy for applications in science, -engineering, and data analysis. - -```{toctree} ---- -maxdepth: 1 ---- - -content/mooreslaw-tutorial -content/tutorial-deep-learning-on-mnist -content/tutorial-x-ray-image-processing -content/tutorial-static_equilibrium -content/tutorial-plotting-fractals -content/tutorial-air-quality-analysis -``` diff --git a/site/articles.md b/site/articles.md deleted file mode 100644 index 8540ed69..00000000 --- a/site/articles.md +++ /dev/null @@ -1,13 +0,0 @@ -# Articles - -```{admonition} Help improve the tutorials! - -Want to make a valuable contribution to the tutorials? Consider working on -these articles so that they become fully executable/reproducible! -``` - -```{toctree} - -content/tutorial-deep-reinforcement-learning-with-pong-from-pixels -content/tutorial-nlp-from-scratch -``` diff --git a/site/contributing.md b/site/contributing.md index e95774a5..a4287490 100644 --- a/site/contributing.md +++ b/site/contributing.md @@ -44,12 +44,6 @@ accept both Jupyter notebooks (`.ipynb`) and MyST-NB notebooks (`.md`). If you want to sync your `.ipynb` to your `.md` file follow the [pairing tutorial](content/pairing). -```{toctree} -:hidden: - -content/pairing -``` - ## Adding your own tutorials If you have your own tutorial in the form of a Jupyter notebook (an `.ipynb` @@ -66,14 +60,7 @@ We will try to respond as quickly as possible with comments, if applicable. ### Check out our suggested template You can use this template to make your content consistent with our existing -tutorials: - -```{toctree} ---- -maxdepth: 1 ---- -content/tutorial-style-guide -``` +tutorials. ### Upload your content diff --git a/site/features.md b/site/features.md deleted file mode 100644 index fb966a8a..00000000 --- a/site/features.md +++ /dev/null @@ -1,13 +0,0 @@ -# NumPy Features - -A collection of notebooks pertaining to built-in NumPy functionality. - -```{toctree} ---- -maxdepth: 1 ---- - -content/tutorial-svd -content/save-load-arrays -content/tutorial-ma -``` diff --git a/site/index.md b/site/index.md index c15d6877..ad8340c2 100644 --- a/site/index.md +++ b/site/index.md @@ -19,18 +19,6 @@ local copy of the `.ipynb` files, you can either [clone this repository](https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/cloning-a-repository) or use the download icon in the upper-right corner of each tutorial. -## Content - -```{toctree} ---- -maxdepth: 2 ---- - -features -applications -contributing -``` - ### Non-executable articles ```{admonition} Help improve the tutorials! @@ -39,14 +27,6 @@ Want to make a valuable contribution to the tutorials? Consider contributing to these existing articles to help make them fully executable and reproducible! ``` -```{toctree} ---- -maxdepth: 2 ---- - -articles -``` - ## Useful links and resources The following links may be useful: From d3721aea5ac0568854e287e87ba9e93422bdf733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 07:10:26 -0800 Subject: [PATCH 121/152] Switching the build engine --- site/Makefile | 29 ----------------------------- site/requirements.txt | 5 +---- tox.ini | 5 +++-- 3 files changed, 4 insertions(+), 35 deletions(-) delete mode 100644 site/Makefile diff --git a/site/Makefile b/site/Makefile deleted file mode 100644 index 89fe53b6..00000000 --- a/site/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -notebooks: - mkdir -p notebooks - jupytext -k python3 ../content/*.md --from myst --to notebook - mv ../content/*.ipynb notebooks - -clean: - rm -rf _build - rm -rf notebooks - -.PHONY: help Makefile notebooks clean - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/site/requirements.txt b/site/requirements.txt index e040deba..e6e749ac 100644 --- a/site/requirements.txt +++ b/site/requirements.txt @@ -1,4 +1 @@ -sphinx -myst-nb -sphinx-book-theme -sphinx-copybutton +jupyter-book>=2 diff --git a/tox.ini b/tox.ini index 34dbfa2e..45c7d916 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ deps = devdeps: matplotlib>=0.0.dev0 devdeps: pandas>=0.0.dev0 -allowlist_externals = bash, make +allowlist_externals = bash commands = # Force numpy reinstall to work around upper version limits in downstream dependencies (e.g. pandas) @@ -42,7 +42,8 @@ commands = !buildhtml: bash -c 'find content -name "*.md" | grep -vf ignore_testing | xargs jupytext --to notebook ' !buildhtml: pytest --nbval-lax --durations=10 content/ - buildhtml: make -C site/ SPHINXOPTS="-nWT --keep-going" html + + buildhtml: bash -c "jupyter-book build --execute --html -d" pip_pre = predeps: true From ab9d00a238b0ba33011de22c5d1759beb742da1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 07:19:52 -0800 Subject: [PATCH 122/152] MAINT: adding to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bee3f8ec..30b4c933 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ pmip tags cscope.out .ipynb_checkpoints +.tox # Compiled source # ################### From 4c6a32c5eda0d373c2a79a6f75e3e81e3e5d6411 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 6 Nov 2025 08:57:33 -0800 Subject: [PATCH 123/152] Attempt at fixing conda job. --- .github/workflows/conda.yml | 8 ++------ environment.yml | 4 ++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 0d2bc0f0..0f519f55 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -40,12 +40,8 @@ jobs: run: | conda info conda list - make -C site/ SPHINXOPTS="-nWT --keep-going" html - - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 - with: - name: sphinx-build-artifact - path: site/_build/html/reports + export OMP_NUM_THREADS=1 + python -m tox -e py313-buildhtml - name: fail on build errors if: steps.build_step.outcome != 'success' diff --git a/environment.yml b/environment.yml index 15dd1085..d22a797e 100644 --- a/environment.yml +++ b/environment.yml @@ -16,3 +16,7 @@ dependencies: - sphinx-copybutton # to load the md files in binder - jupytext + - pip + - pip: + - jupyterbook>=2 + - jupyterlab_myst From 2ec2841795a3d4e18f847688288b82d6f0935c4d Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 6 Nov 2025 09:02:28 -0800 Subject: [PATCH 124/152] Spelling. --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d22a797e..8cfda206 100644 --- a/environment.yml +++ b/environment.yml @@ -18,5 +18,5 @@ dependencies: - jupytext - pip - pip: - - jupyterbook>=2 + - jupyter-book>=2 - jupyterlab_myst From 6ab890fdc0a3ca9d13736fb6ba89e9ddeddd421e Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Thu, 6 Nov 2025 09:05:53 -0800 Subject: [PATCH 125/152] Add tox, rm sphinx. --- environment.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/environment.yml b/environment.yml index 8cfda206..769fb7b0 100644 --- a/environment.yml +++ b/environment.yml @@ -10,11 +10,7 @@ dependencies: - pandas - imageio # For building the site - - sphinx - - myst-nb - - sphinx-book-theme - - sphinx-copybutton - # to load the md files in binder + - tox - jupytext - pip - pip: From deb2344487d9b9e02c30da70619c147cdbb1a2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 19:56:55 -0800 Subject: [PATCH 126/152] CI: fixing conda job --- .github/workflows/conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 0f519f55..662289ff 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -41,7 +41,7 @@ jobs: conda info conda list export OMP_NUM_THREADS=1 - python -m tox -e py313-buildhtml + jupyter-book build --execute --html -d - name: fail on build errors if: steps.build_step.outcome != 'success' From 09fe7d6f150686fce9f6afa7b6145ecb427a33e5 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 7 Nov 2025 00:06:16 -0800 Subject: [PATCH 127/152] CI: rm conda job - to be updated with pixi later. --- .github/workflows/conda.yml | 48 ------------------------------------- 1 file changed, 48 deletions(-) delete mode 100644 .github/workflows/conda.yml diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml deleted file mode 100644 index 662289ff..00000000 --- a/.github/workflows/conda.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Test building site with conda environment - -on: - push: - branches: - - main - pull_request: - branches: - - main - schedule: - - cron: '0 5 * * 1' - workflow_dispatch: - -jobs: - test: - runs-on: ${{ matrix.os }}-latest - - strategy: - matrix: - os: [ubuntu, macos, windows] - - defaults: - run: - shell: bash -l {0} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 - with: - auto-update-conda: true - activate-environment: numpy-tutorials - environment-file: environment.yml - miniforge-version: latest - use-mamba: true - python-version: "3.11" - auto-activate-base: false - - name: inspect and build - id: build_step - continue-on-error: true - run: | - conda info - conda list - export OMP_NUM_THREADS=1 - jupyter-book build --execute --html -d - - - name: fail on build errors - if: steps.build_step.outcome != 'success' - run: exit 1 From 9344931d20ba9ec531da4ace8b70df6c1dde76fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Sat, 8 Nov 2025 08:12:21 -0800 Subject: [PATCH 128/152] =?UTF-8?q?CI:=20pass=20on=20the=20BASE=5FURL=20?= =?UTF-8?q?=F0=9F=A4=A6=F0=9F=A4=A6=E2=80=8D=E2=99=80=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 45c7d916..8ec29880 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ description = run tests setenv = devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple -passenv = OMP_NUM_THREADS +passenv = BASE_URL, OMP_NUM_THREADS deps = # We use these files to specify all the dependencies, and below we override From bc4cb6dc8cc7694a024e76e7ccc2a8132cbb5d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 6 Nov 2025 07:41:59 -0800 Subject: [PATCH 129/152] Adding GHA deployment for the rendered pages --- .github/workflows/ci_publish.yml | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/ci_publish.yml diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml new file mode 100644 index 00000000..510284af --- /dev/null +++ b/.github/workflows/ci_publish.yml @@ -0,0 +1,46 @@ +name: Build and Publish HTML and deployed_notebooks + +on: + push: + branches: + - main + schedule: + - cron: '0 5 * * 1' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # `BASE_URL` determines, relative to the root of the domain, the URL that your site is served from. + # E.g., if your site lives at `https://mydomain.org/myproject`, set `BASE_URL=/myproject`. + # If, instead, your site lives at the root of the domain, at `https://mydomain.org`, set `BASE_URL=''`. + BASE_URL: /${{ github.event.repository.name }} + +jobs: + + publish_html: + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + name: Publish HTML + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install dependencies + run: python -m pip install --upgrade tox + + - name: Execute notebooks while building HTMLs + run: tox -e py312-buildhtml + + - name: Publish + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./_build/html/ + commit_message: ${{ github.event.head_commit.message }} From 08fcaf9f8e56dc30a8d4e5420c651e460e75303b Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 14 Nov 2025 11:10:17 -0800 Subject: [PATCH 130/152] Begin incorporating myst features into notebooks (#279) * Use note directive. * Use tip directive. * Use note directive. * Use note directive. * Use raises-exception tag. * Use note directive. * Suppress imshow warning. --- content/tutorial-svd.md | 55 +++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index c414cafe..d2c0adb6 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -43,7 +43,18 @@ from scipy.datasets import face img = face() ``` -**Note**: If you prefer, you can use your own image as you work through this tutorial. In order to transform your image into a NumPy array that can be manipulated, you can use the `imread` function from the [matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. Alternatively, you can use the [imageio.imread](https://imageio.readthedocs.io/en/stable/_autosummary/imageio.v3.imread.html) function from the `imageio` library. Be aware that if you use your own image, you'll likely need to adapt the steps below. For more information on how images are treated when converted to NumPy arrays, see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. +```{note} +If you prefer, you can use your own image as you work through this tutorial. +In order to transform your image into a NumPy array that can be manipulated, you +can use the `imread` function from the +[matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. +Alternatively, you can use the +[imageio.imread](https://imageio.readthedocs.io/en/stable/_autosummary/imageio.v3.imread.html) +function from the `imageio` library. +Be aware that if you use your own image, you'll likely need to adapt the steps below. +For more information on how images are treated when converted to NumPy arrays, +see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. +``` +++ @@ -105,7 +116,13 @@ Since we are going to perform linear algebra operations on this data, it might b img_array = img / 255 ``` -This operation, dividing an array by a scalar, works because of NumPy's [broadcasting rules](https://numpy.org/devdocs/user/theory.broadcasting.html#array-broadcasting-in-numpy). (Note that in real-world applications, it would be better to use, for example, the [img_as_float](https://scikit-image.org/docs/stable/api/skimage.html#skimage.img_as_float) utility function from `scikit-image`). +This operation, dividing an array by a scalar, works because of NumPy's [broadcasting rules](https://numpy.org/devdocs/user/theory.broadcasting.html#array-broadcasting-in-numpy). + +```{tip} +In real-world applications, it may be better to use, for example, the +[img_as_float](https://scikit-image.org/docs/stable/api/skimage.html#skimage.img_as_float) +utility function from `scikit-image`. +``` You can check that the above works by doing some tests; for example, inquiring about maximum and minimum values for this array: @@ -134,7 +151,19 @@ It is possible to use methods from linear algebra to approximate an existing set +++ -**Note**: We will use NumPy's linear algebra module, [numpy.linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg), to perform the operations in this tutorial. Most of the linear algebra functions in this module can also be found in [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg), and users are encouraged to use the [scipy](https://docs.scipy.org/doc/scipy/reference/index.html#module-scipy) module for real-world applications. However, some functions in the [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg) module, such as the SVD function, only support 2D arrays. For more information on this, check the [scipy.linalg page](https://docs.scipy.org/doc/scipy/tutorial/linalg.html). +```{note} +We will use NumPy's linear algebra module, +[numpy.linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg), +to perform the operations in this tutorial. +Most of the linear algebra functions in this module can also be found in +[scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg), +and users are encouraged to use the [scipy](https://docs.scipy.org/doc/scipy/reference/index.html#module-scipy) +module for real-world applications. +However, some functions in the +[scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg) +module, such as the SVD function, only support 2D arrays. +For more information on this, check the [scipy.linalg page](https://docs.scipy.org/doc/scipy/tutorial/linalg.html). +``` +++ @@ -177,7 +206,11 @@ import numpy as np U, s, Vt = np.linalg.svd(img_gray) ``` -**Note** If you are using your own image, this command might take a while to run, depending on the size of your image and your hardware. Don't worry, this is normal! The SVD can be a pretty intensive computation. +```{note} +If you are using your own image, this command might take a while to run, +depending on the size of your image and your hardware. +Don't worry, this is normal! The SVD can be a pretty intensive computation. +``` +++ @@ -188,9 +221,10 @@ U.shape, s.shape, Vt.shape ``` Note that `s` has a particular shape: it has only one dimension. This means that some linear algebra functions that expect 2d arrays might not work. For example, from the theory, one might expect `s` and `Vt` to be -compatible for multiplication. However, this is not true as `s` does not have a second axis. Executing +compatible for multiplication. However, this is not true as `s` does not have a second axis: -```python +```{code-cell} +:tags: [raises-exception] s @ Vt ``` @@ -331,7 +365,12 @@ plt.imshow(np.transpose(reconstructed, (1, 2, 0))) plt.show() ``` -In fact, `imshow` peforms this clipping under-the-hood, so if you skip the first line in the previous code cell, you might see a warning message saying `"Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers)."` +```{note} +In fact, `imshow` peforms this clipping under-the-hood, so if you skip the first +line in the previous code cell, you might see a warning message saying +`"Clipping input data to the valid range for imshow with RGB data ([0..1] for +floats or [0..255] for integers)."` +``` Now, to do the approximation, we must choose only the first `k` singular values for each color channel. This can be done using the following syntax: @@ -351,7 +390,7 @@ approx_img.shape which is not the right shape for showing the image. Finally, reordering the axes back to our original shape of `(768, 1024, 3)`, we can see our approximation: ```{code-cell} -plt.imshow(np.transpose(approx_img, (1, 2, 0))) +plt.imshow(np.transpose(np.clip(approx_img, 0, 1), (1, 2, 0))) plt.show() ``` From 558d73ed9ded600c4ae3b72dfefec60f403c0930 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 14 Nov 2025 12:05:46 -0800 Subject: [PATCH 131/152] Rm pairing article. --- content/pairing.md | 247 --------------------------------------------- 1 file changed, 247 deletions(-) delete mode 100644 content/pairing.md diff --git a/content/pairing.md b/content/pairing.md deleted file mode 100644 index 322fb462..00000000 --- a/content/pairing.md +++ /dev/null @@ -1,247 +0,0 @@ ---- -jupytext: - formats: ipynb,md:myst - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.11.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -# Pairing Jupyter notebooks and MyST-NB - -## What you'll do -This guide will keep a Jupyter notebook synced _or paired_ between -`.ipynb` and `.md`. - -## What you'll learn -- The difference between Jupyter's json format and MyST-NB's markdown - format -- The benefits and drawbacks of json and markdown -- How to keep `.ipynb` and `.md` files in sync - -## What you'll need -- [Jupyter](https://jupyter.org/) -- [Jupytext](https://jupytext.readthedocs.io/en/latest/index.html) - ---- -## Background - -The [NumPy tutorials](https://github.com/numpy/numpy-tutorials) are -reviewed and executed as [MyST-NB](https://myst-nb.readthedocs.io/) -notebooks. Content is easier to review in this markdown format. You can -keep your `.ipynb` in sync with the content on NumPy tutorials. The -NumPy tutorials use -[Jupytext](https://jupytext.readthedocs.io/en/latest/index.html) to -convert your `.ipynb` file to [MyST -Markdown](https://jupytext.readthedocs.io/en/latest/formats-markdown.html) -format. - -Jupyter notebooks are stored on your disk in a -[json format](https://nbformat.readthedocs.io/en/latest/format_description.html). The json format is -very powerful and allows you to store almost any input and output that -Python libraries can create. The drawback is that it is hard to see and compare changes made in the notebook file when reviewing pull requests, because this means the reviewers are looking only at the raw json files. - -MyST-NB notebooks are stored on your disk in a -[markdown](https://en.wikipedia.org/wiki/Markdown) format. The markdown -format is a lightweight markup language. Its key design goal is -[_readability_](https://daringfireball.net/projects/markdown/syntax#philosophy). -The drawback is that markdown can only store the inputs of your code. -Each time you open the notebook, you must execute the inputs to see the -output. - -> __Note:__ You should use [common mark](https://commonmark.org) -> markdown cells. Jupyter only renders common mark markdown, but MyST-NB -> supports a variety of restructured text directives. These Sphinx -> markdown directives will render when NumPy tutorials are built into a -> static website, but they will show up as raw code when you open in -> Jupyter locally or on [Binder](https://mybinder.org). - -Consider these two versions of the same __Simple notebook example__. You -have three things in the notebooks: - -1. A markdown cell that explains the code - ```This code calculates 2+2 and prints the output.``` -2. A code cell that shows the code - ```python - x = 2 + 2 - print('x = ', x) - ``` -3. The output of the code cell - ```python - x = 4 - ``` ---- -__
Simple notebook example
__ -This code calculates 2+2 and prints the output. - -```{code-cell} -x = 2 + 2 -print("x = ", x) -``` - ---- - -Here are the two Simple notebook example raw inputs side-by-side: - - - - - - - - - - - -
json .ipynbMyST-NB .md
- -```json -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This code calculates 2+2 and prints the output" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "x = 4\n" - ] - } - ], - "source": [ - "x = 2 + 2\n", - "print('x = ', x)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} -``` - - - -```` ---- -jupytext: - formats: ipynb,md:myst - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.6.0 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -This code calculates 2+2 and prints the output - -```{code-cell} ipython3 -x = 2 + 2 -print('x = ', x) -``` -```` -
- -The MyST-NB `.md` is much shorter, but it does not save the output `4`. - - -## Pair your notebook files `.ipynb` and `.md` - -When you submit a Jupyter notebook to NumPy tutorials, we (the reviewers) will convert -it to a MyST-NB format. You can also submit the MyST-NB `.md` in your -pull request. -To keep the `.ipynb` and `.md` in sync--_or paired_--you need -[Jupytext](https://jupytext.readthedocs.io/en/latest/index.html). - -Install `jupytext` using: - -``` -pip install jupytext -``` -or -``` -conda install jupytext -c conda-forge -``` - -Once installed, start your `jupyter lab` or `jupyter notebook` -session in the browser. When launching `jupyter lab` it will ask you to rebuild -to include the Jupytext extension. - -You can pair the two formats in the classic Jupyter, Jupyter Lab, -or the command line: - -```{admonition} **1. Classic Jupyter Jupytext pairing** -:class: toggle - -![Animation showing pairing with Jupyter classic](_static/01-classic.gif) -``` - -```{admonition} **2. JupyterLab Jupytext pairing** -:class: toggle - -![Animation showing pairing with JupyterLab](_static/02-jupyterlab.gif) -``` - -````{admonition} **3. Command line Jupytext pairing** -:class: toggle - -```sh -jupytext --set-formats ipynb,myst notebook.ipynb -``` - -Then, update either the MyST markdown or notebook file: - -```sh -jupytext --sync notebook.ipynb -``` -```` - -> __Note:__ With Jupytext installed, the classic Jupyter interface will -> automatically open MyST files as notebooks. In JupyterLab, you can -> right-click and choose "Open With -> Notebook" to open as a notebook. -> The outputs of your code cells are only saved in the `.ipynb` file. - -## Wrapping up - -In this tutorial, you saw the json `.ipynb` and MyST-NB `.md` raw code -to create Jupyter notebooks. You can use both formats to create -tutorials. Now you can work in either a simple text editor like VIM -or emacs or continue building notebooks in your browser. Jupytext can -handle pairing to keep your work in sync. From 98e5878d3fa1aee2453ed4ca0b8deeca541d54bf Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 14 Nov 2025 12:09:19 -0800 Subject: [PATCH 132/152] Replace pairing references with links to jupyterlab_myst. --- README.md | 7 ++++--- site/contributing.md | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b5453a6e..9c387a85 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ used in the main NumPy documentation has two reasons: You may notice our content is in markdown format (`.md` files). We review and host notebooks in the [MyST-NB](https://myst-nb.readthedocs.io/) format. We -accept both Jupyter notebooks (`.ipynb`) and MyST-NB notebooks (`.md`). If you want -to sync your `.ipynb` to your `.md` file follow the [pairing -tutorial](content/pairing.md). +accept both Jupyter notebooks (`.ipynb`) and MyST-NB notebooks (`.md`). +If you want to author `MyST` notebooks in jupyterlab, check out the +[jupyterlab_myst](https://mystmd.org/guide/quickstart-jupyter-lab-myst) extension! + ### Adding your own tutorials diff --git a/site/contributing.md b/site/contributing.md index a4287490..9fd31d4b 100644 --- a/site/contributing.md +++ b/site/contributing.md @@ -41,8 +41,8 @@ used in the main NumPy documentation has two reasons: You may notice our content is in markdown format (`.md` files). We review and host notebooks in the [MyST-NB](https://myst-nb.readthedocs.io/) format. We accept both Jupyter notebooks (`.ipynb`) and MyST-NB notebooks (`.md`). -If you want to sync your `.ipynb` to your `.md` file follow the [pairing -tutorial](content/pairing). +If you want to author `MyST` notebooks in jupyterlab, check out the +[jupyterlab_myst](https://mystmd.org/guide/quickstart-jupyter-lab-myst) extension! ## Adding your own tutorials From 2d2ced4fb4379a6e77e3ab885f6a40ebe1895835 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 14 Nov 2025 12:09:46 -0800 Subject: [PATCH 133/152] Rm pairing from meta test spec. --- ignore_testing | 1 - 1 file changed, 1 deletion(-) diff --git a/ignore_testing b/ignore_testing index a72bcd51..e5b7e003 100644 --- a/ignore_testing +++ b/ignore_testing @@ -1,4 +1,3 @@ content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md -content/pairing.md content/tutorial-style-guide.md content/tutorial-nlp-from-scratch.md From c1afea5c880e281690e1fc32408b581569b9c7af Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Fri, 14 Nov 2025 12:43:42 -0800 Subject: [PATCH 134/152] Rm un-executable nlp tutorial. --- content/tutorial-nlp-from-scratch.md | 1061 ----------------- content/tutorial-nlp-from-scratch/README.md | 68 -- .../dl_architectures.jpg | Bin 25958 -> 0 bytes .../tutorial-nlp-from-scratch/speeches.csv | 113 -- 4 files changed, 1242 deletions(-) delete mode 100644 content/tutorial-nlp-from-scratch.md delete mode 100644 content/tutorial-nlp-from-scratch/README.md delete mode 100644 content/tutorial-nlp-from-scratch/dl_architectures.jpg delete mode 100644 content/tutorial-nlp-from-scratch/speeches.csv diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md deleted file mode 100644 index c5809cdb..00000000 --- a/content/tutorial-nlp-from-scratch.md +++ /dev/null @@ -1,1061 +0,0 @@ ---- -jupyter: - jupytext: - formats: md,ipynb - text_representation: - extension: .md - format_name: markdown - format_version: '1.3' - jupytext_version: 1.11.5 - kernelspec: - display_name: Python 3 (ipykernel) - language: python - name: python3 ---- - -# Sentiment Analysis on notable speeches of the last decade - -```{caution} - -This article is not currently tested. Help improve this tutorial by making it -fully executable! -``` - -This tutorial demonstrates how to build a simple Long Short Term memory network (LSTM) from scratch in NumPy to perform sentiment analysis on a socially relevant and ethically acquired dataset. - -Your deep learning model (the LSTM) is a form of a Recurrent Neural Network and will learn to classify a piece of text as positive or negative from the IMDB reviews dataset. The dataset contains 50,000 movie reviews and corresponding labels. Based on the numeric representations of these reviews and their corresponding labels (supervised learning) the neural network will be trained to learn the sentiment using forward propagation and backpropagation through time since we are dealing with sequential data here. The output will be a vector containing the probabilities that the text samples are positive. - - -Today, Deep Learning is getting adopted in everyday life and now it is more important to ensure that decisions that have been taken using AI are not reflecting discriminatory behavior towards a set of populations. It is important to take fairness into consideration while consuming the output from AI. Throughout the tutorial we'll try to question all the steps in our pipeline from an ethics point of view. - - -## Prerequisites - -You are expected to be familiar with the Python programming language and array manipulation with NumPy. In addition, some understanding of Linear Algebra and Calculus is recommended. You should also be familiar with how Neural Networks work. For reference, you can visit the [Python](https://docs.python.org/dev/tutorial/index.html), [Linear algebra on n-dimensional arrays](https://numpy.org/numpy-tutorials/content/tutorial-svd.html) and [Calculus](https://d2l.ai/chapter_appendix-mathematics-for-deep-learning/multivariable-calculus.html) tutorials. - -To get a refresher on Deep Learning basics, You should consider reading [the d2l.ai book](https://d2l.ai/chapter_recurrent-neural-networks/index.html), which is an interactive deep learning book with multi-framework code, math, and discussions. You can also go through the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) to understand how a basic neural network is implemented from scratch. - -In addition to NumPy, you will be utilizing the following Python standard modules for data loading and processing: -- [`pandas`](https://pandas.pydata.org/docs/) for handling dataframes -- [`Matplotlib`](https://matplotlib.org/) for data visualization -- [`pooch`](https://www.fatiando.org/pooch/latest/https://www.fatiando.org/pooch/latest/) to download and cache datasets - -This tutorial can be run locally in an isolated environment, such as [Virtualenv](https://virtualenv.pypa.io/en/stable/) or [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html). You can use [Jupyter Notebook or JupyterLab](https://jupyter.org/install) to run each notebook cell. - - -## Table of contents - -1. Data Collection - -2. Preprocess the datasets - -3. Build and train an LSTM network from scratch - -4. Perform sentiment analysis on collected speeches - -5. Next steps - - -## 1. Data Collection - -Before you begin there are a few pointers you should always keep in mind before choosing the data you wish to train your model on: -- **Identifying Data Bias** - Bias is an inherent component of the human thought process. Therefore data sourced from human activities reflects that bias. Some ways in which this bias tends to occur in Machine Learning datasets are: - - *Bias in historical data*: Historical data are often skewed towards, or against, particular groups. - Data can also be severely imbalanced with limited information on protected groups. - - *Bias in data collection mechanisms*: Lack of representativeness introduces inherent biases in the data collection process. - - *Bias towards observable outcomes*: In some scenarios, we have the information about True Outcomes only for a certain section of the population. In the absence of information on all outcomes, one cannot even measure fairness -- **Preserving human anonymity for sensitive data**: [Trevisan and Reilly](https://eprints.whiterose.ac.uk/91157/1/Ethical%20dilemmas.pdf) identified a list of sensitive topics that need to be handled with extra care. We present the same below along with a few additions: - - personal daily routines (including location data); - - individual details about impairment and/or medical records; - - emotional accounts of pain and chronic illness; - - financial information about income and/or welfare payments; - - discrimination and abuse episodes; - - criticism/praise of individual providers of healthcare and support services; - - suicidal thoughts; - - criticism/praise of a power structure especially if it compromises their safety; - - personally-identifying information (even if anonymized in some way) including things like fingerprints or voice. - ->While it can be difficult taking consent from so many people especially on on-line platforms, the necessity of it depends upon the sensitivity of the topics your data includes and other indicators like whether the platform the data was obtained from allows users to operate under pseudonyms. If the website has a policy that forces the use of a real name, then the users need to be asked for consent. - -In this section, you will be collecting two different datasets: the IMDb movie reviews dataset, and a collection of 10 speeches curated for this tutorial including activists from different countries around the world, different times, and different topics. The former would be used to train the deep learning model while the latter will be used to perform sentiment analysis on. - - -### Collecting the IMDb reviews dataset -IMDb Reviews Dataset is a large movie review dataset collected and prepared by Andrew L. Maas from the popular movie rating service, IMDb. The IMDb Reviews dataset is used for binary sentiment classification, whether a review is positive or negative. It contains 25,000 movie reviews for training and 25,000 for testing. All these 50,000 reviews are labeled data that may be used for supervised deep learning. For ease of reproducibility, we'll be sourcing the data from [Zenodo](https://zenodo.org/record/4117827#.YVQZ_EZBy3Ihttps://zenodo.org/record/4117827#.YVQZ_EZBy3I). - > The IMDb platform allows the usage of their public datasets for personal and non-commercial use. We did our best to ensure that these reviews do not contain any of the aforementioned sensitive topics pertaining to the reviewer. - - -### Collecting and loading the speech transcripts -We have chosen speeches by activists around the globe talking about issues like climate change, feminism, lgbtqa+ rights and racism. These were sourced from newspapers, the official website of the United Nations and the archives of established universities as cited in the table below. A CSV file was created containing the transcribed speeches, their speaker and the source the speeches were obtained from. -We made sure to include different demographics in our data and included a range of different topics, most of which focus on social and/or ethical issues. - -| Speech | Speaker | Source | -|--------------------------------------------------|-------------------------|------------------------------------------------------------| -| Barnard College Commencement | Leymah Gbowee | [Barnard College](https://barnard.edu/news/transcript-speech-nobel-peace-prize-winner-leymah-gbowee) | -| UN Speech on youth Education | Malala Yousafzai | [The Guardian](https://www.theguardian.com/commentisfree/2013/jul/12/malala-yousafzai-united-nations-education-speech-text) | -| Remarks in the UNGA on racial discrimination | Linda Thomas Greenfield | [United States mission to the United Nation](https://usun.usmission.gov/remarks-by-ambassador-linda-thomas-greenfield-at-a-un-general-assembly-commemorative-meeting-for-intl-day-for-the-elimination-of-racial-discrimination/) | -| How Dare You | Greta Thunberg | [NBC](https://www.nbcnews.com/news/world/read-greta-thunberg-s-full-speech-united-nations-climate-action-n1057861) | -| The speech that silenced the world for 5 minutes | Severn Suzuki | [Earth Charter](https://earthcharter.org/new-voices-after-26-years-of-the-girl-who-silenced-the-world-for-5-minutes/) | -| The Hope Speech | Harvey Milk | [Museum of Fine Arts, Boston](https://www.mfa.org/exhibitions/amalia-pica/transcript-harvey-milks-the-hope-speech) | -| Speech at the time to Thrive Conference | Ellen Page | [Huffpost](https://www.huffpost.com/entry/time-to-thrive_b_4794251) | -| I have a dream | Martin Luther King | [Marshall University](https://www.marshall.edu/onemarshallu/i-have-a-dream/) | - - -## 2. Preprocess the datasets ->Preprocessing data is an extremely crucial step before building any Deep learning model, however in an attempt to keep the tutorial focused on building the model, we will not dive deep into the code for preprocessing. Given below is a brief overview of all the steps we undertake to clean our data and convert it to its numeric representation. - -1. **Text Denoising** : Before converting your text into vectors, it is important to clean it and remove all unhelpful parts a.k.a. the noise from your data by converting all characters to lowercase, removing html tags, brackets and stop words (words that don't add much meaning to a sentence). Without this step the dataset is often a cluster of words that the computer doesn't understand. - - -2. **Converting words to vectors** : A word embedding is a learned representation for text where words that have the same meaning have a similar representation. Individual words are represented as real-valued vectors in a predefined vector space. GloVe is an unsupervised algorithm developed by Stanford for generating word embeddings by generating global word-word co-occurence matrix from a corpus. You can download the zipped files containing the embeddings from [the GloVe official website](https://nlp.stanford.edu/projects/glove/). Here you can choose any of the four options for different sizes or training datasets. We have chosen the least memory consuming embedding file. - >The GloVe word embeddings include sets that were trained on billions of tokens, some up to 840 billion tokens. These algorithms exhibit stereotypical biases, such as gender bias which can be traced back to the original training data. For example certain occupations seem to be more biased towards a particular gender, reinforcing problematic stereotypes. The nearest solution to this problem are some de-biasing algorithms as the one presented in [this research article](https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1184/reports/6835575.pdf), which one can use on embeddings of their choice to mitigate bias, if present. - - -You'll start with importing the necessary packages to build our Deep Learning network. - -```python -# Importing the necessary packages -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import pooch -import string -import re -import zipfile -import os - -# Creating the random instance -rng = np.random.default_rng() -``` - -Next, you'll define set of text preprocessing helper functions. - -```python -class TextPreprocess: - """Text Preprocessing for a Natural Language Processing model.""" - - def txt_to_df(self, file): - """Function to convert a txt file to pandas dataframe. - - Parameters - ---------- - file : str - Path to the txt file. - - Returns - ------- - Pandas dataframe - txt file converted to a dataframe. - - """ - with open(imdb_train, 'r') as in_file: - stripped = (line.strip() for line in in_file) - reviews = {} - for line in stripped: - lines = [splits for splits in line.split("\t") if splits != ""] - reviews[lines[1]] = float(lines[0]) - df = pd.DataFrame(reviews.items(), columns=['review', 'sentiment']) - df = df.sample(frac=1).reset_index(drop=True) - return df - - def unzipper(self, zipped, to_extract): - """Function to extract a file from a zipped folder. - - Parameters - ---------- - zipped : str - Path to the zipped folder. - - to_extract: str - Path to the file to be extracted from the zipped folder - - Returns - ------- - str - Path to the extracted file. - - """ - fh = open(zipped, 'rb') - z = zipfile.ZipFile(fh) - outdir = os.path.split(zipped)[0] - z.extract(to_extract, outdir) - fh.close() - output_file = os.path.join(outdir, to_extract) - return output_file - - def cleantext(self, df, text_column=None, - remove_stopwords=True, remove_punc=True): - """Function to clean text data. - - Parameters - ---------- - df : pandas dataframe - The dataframe housing the input data. - text_column : str - Column in dataframe whose text is to be cleaned. - remove_stopwords : bool - if True, remove stopwords from text - remove_punc : bool - if True, remove punctuation symbols from text - - Returns - ------- - Numpy array - Cleaned text. - - """ - # converting all characters to lowercase - df[text_column] = df[text_column].str.lower() - - # List of stopwords taken from https://gist.github.com/sebleier/554280 - stopwords = ["a", "about", "above", "after", "again", "against", - "all", "am", "an", "and", "any", "are", - "as", "at", "be", "because", - "been", "before", "being", "below", - "between", "both", "but", "by", "could", - "did", "do", "does", "doing", "down", "during", - "each", "few", "for", "from", "further", - "had", "has", "have", "having", "he", - "he'd", "he'll", "he's", "her", "here", - "here's", "hers", "herself", "him", - "himself", "his", "how", "how's", "i", - "i'd", "i'll", "i'm", "i've", - "if", "in", "into", - "is", "it", "it's", "its", - "itself", "let's", "me", "more", - "most", "my", "myself", "nor", "of", - "on", "once", "only", "or", - "other", "ought", "our", "ours", - "ourselves", "out", "over", "own", "same", - "she", "she'd", "she'll", "she's", "should", - "so", "some", "such", "than", "that", - "that's", "the", "their", "theirs", "them", - "themselves", "then", "there", "there's", - "these", "they", "they'd", "they'll", - "they're", "they've", "this", "those", - "through", "to", "too", "under", "until", "up", - "very", "was", "we", "we'd", "we'll", - "we're", "we've", "were", "what", - "what's", "when", "when's", - "where", "where's", - "which", "while", "who", "who's", - "whom", "why", "why's", "with", - "would", "you", "you'd", "you'll", - "you're", "you've", - "your", "yours", "yourself", "yourselves"] - - def remove_stopwords(data, column): - data[f'{column} without stopwords'] = data[column].apply( - lambda x: ' '.join([word for word in x.split() if word not in (stopwords)])) - return data - - def remove_tags(string): - result = re.sub('<*>', '', string) - return result - - # remove html tags and brackets from text - if remove_stopwords: - data_without_stopwords = remove_stopwords(df, text_column) - data_without_stopwords[f'clean_{text_column}'] = data_without_stopwords[f'{text_column} without stopwords'].apply( - lambda cw: remove_tags(cw)) - if remove_punc: - data_without_stopwords[f'clean_{text_column}'] = data_without_stopwords[f'clean_{text_column}'].str.replace( - '[{}]'.format(string.punctuation), ' ', regex=True) - - X = data_without_stopwords[f'clean_{text_column}'].to_numpy() - - return X - - - def sent_tokeniser(self, x): - """Function to split text into sentences. - - Parameters - ---------- - x : str - piece of text - - Returns - ------- - list - sentences with punctuation removed. - - """ - sentences = re.split(r'(? Since we will be performing paragraph wise sentiment analysis on each speech further ahead in the tutorial, we'll need the punctuation marks to split the text into paragraphs, hence we refrain from removing their punctuation marks at this stage. - -```python -speech_data_path = 'tutorial-nlp-from-scratch/speeches.csv' -speech_df = pd.read_csv(speech_data_path) -X_pred = textproc.cleantext(speech_df, - text_column='speech', - remove_stopwords=True, - remove_punc=False) -speakers = speech_df['speaker'].to_numpy() -``` - -You will now download the `GloVe` embeddings, unzip them and build a dictionary mapping each word and word embedding. This will act as a cache for when you need to replace each word with its respective word embedding. - -```python -glove = data.fetch('glove.6B.50d.zip') -emb_path = textproc.unzipper(glove, 'glove.6B.300d.txt') -emb_matrix = textproc.loadGloveModel(emb_path) -``` - -## 3. Build the Deep Learning Model - It is time to start implementing our LSTM! You will have to first familiarize yourself with some high-level concepts of the basic building blocks of a deep learning model. You can refer to the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) for the same. - -You will then learn how a Recurrent Neural Network differs from a plain Neural Network and what makes it so suitable for processing sequential data. Afterwards, you will construct the building blocks of a simple deep learning model in Python and NumPy and train it to learn to classify the sentiment of a piece of text as positive or negative with a certain level of accuracy. - -### Introduction to a Long Short Term Memory Network - -In a [Multilayer perceptron](https://en.wikipedia.org/wiki/Multilayer_perceptron) (MLP), the information only moves in one direction — from the input layer, through the hidden layers, to the output layer. The information moves straight through the network and never takes the previous nodes into account at a later stage. Because it only considers the current input, the features learned are not shared across different positions of the sequence. Moreover, it cannot process sequences with varying lengths. - -Unlike an MLP, the RNN was designed to work with sequence prediction problems. RNNs introduce state variables to store past information, together with the current inputs, to determine the current outputs. Since an RNN shares the learned features with all the data points in a sequence regardless of its length, it is capable of processing sequences with varying lengths. - -The problem with an RNN however, is that it cannot retain long-term memory because the influence of a given input on the hidden layer, and therefore on the network output, either decays or blows up exponentially as it cycles around the network’s recurrent connections. This shortcoming is referred to as the vanishing gradient problem. Long Short-Term Memory (LSTM) is an RNN architecture specifically designed to address the [vanishing gradient problem](https://en.wikipedia.org/wiki/Vanishing_gradient_problem). - - -### Overview of the Model Architecture - -![Overview of the model architecture, showing a series of animated boxes. There are five identical boxes labeled A and receiving as input one of the words in the phrase "life's a box of chocolates". Each box is highlighted in turn, representing the memory blocks of the LSTM network as information passes through them, ultimately reaching a "Positive" output value.](_static/lstm.gif) - -In the above gif, the rectangles labeled $A$ are called `Cells` and they are the **Memory Blocks** of our LSTM network. They are responsible for choosing what to remember in a sequence and pass on that information to the next cell via two states called the `hidden state` $H_{t}$ and the `cell state` $C_{t}$ where $t$ indicates the time-step. Each `Cell` has dedicated gates which are responsible for storing, writing or reading the information passed to an LSTM. You will now look closely at the architecture of the network by implementing each mechanism happening inside of it. - - -Let's start with writing a function to randomly initialize the parameters which will be learned while our model trains: - -```python -def initialise_params(hidden_dim, input_dim): - # forget gate - Wf = rng.standard_normal(size=(hidden_dim, hidden_dim + input_dim)) - bf = rng.standard_normal(size=(hidden_dim, 1)) - # input gate - Wi = rng.standard_normal(size=(hidden_dim, hidden_dim + input_dim)) - bi = rng.standard_normal(size=(hidden_dim, 1)) - # candidate memory gate - Wcm = rng.standard_normal(size=(hidden_dim, hidden_dim + input_dim)) - bcm = rng.standard_normal(size=(hidden_dim, 1)) - # output gate - Wo = rng.standard_normal(size=(hidden_dim, hidden_dim + input_dim)) - bo = rng.standard_normal(size=(hidden_dim, 1)) - - # fully connected layer for classification - W2 = rng.standard_normal(size=(1, hidden_dim)) - b2 = np.zeros((1, 1)) - - parameters = { - "Wf": Wf, - "bf": bf, - "Wi": Wi, - "bi": bi, - "Wcm": Wcm, - "bcm": bcm, - "Wo": Wo, - "bo": bo, - "W2": W2, - "b2": b2 - } - return parameters -``` - -### Forward Propagation - -Now that you have your initialized parameters, you can pass the input data in a forward direction through the network. Each layer accepts the input data, processes it and passes it to the successive layer. This process is called `Forward Propagation`. You will undertake the following mechanism to implement it: -- Loading the word embeddings of the input data -- Passing the embeddings to an LSTM -- Perform all the gate mechanisms in every memory block of the LSTM to obtain the final hidden state -- Passing the final hidden state through a fully connected layer to obtain the probability with which the sequence is positive -- Storing all the calculated values in a cache to utilize during backpropagation - - -[Sigmoid](https://d2l.ai/chapter_multilayer-perceptrons/mlp.html?highlight=sigmoid#sigmoid-function) belongs to the family of non-linear activation functions. It helps the network to update or forget the data. If the sigmoid of a value results in 0, the information is considered forgotten. Similarly, the information stays if it is 1. - -```python -def sigmoid(x): - n = np.exp(np.fmin(x, 0)) - d = (1 + np.exp(-np.abs(x))) - return n / d -``` - -The **Forget Gate** takes the current word embedding and the previous hidden state concatenated together as input. and decides what parts of the old memory cell content need attention and which can be ignored. - -```python -def fp_forget_gate(concat, parameters): - ft = sigmoid(np.dot(parameters['Wf'], concat) - + parameters['bf']) - return ft -``` - -The **Input Gate** takes the current word embedding and the previous hidden state concatenated together as input. and governs how much of the new data we take into account via the **Candidate Memory Gate** which utilizes the [Tanh](https://d2l.ai/chapter_multilayer-perceptrons/mlp.html?highlight=tanh#tanh-function) to regulate the values flowing through the network. - -```python -def fp_input_gate(concat, parameters): - it = sigmoid(np.dot(parameters['Wi'], concat) - + parameters['bi']) - cmt = np.tanh(np.dot(parameters['Wcm'], concat) - + parameters['bcm']) - return it, cmt -``` - -Finally we have the **Output Gate** which takes information from the current word embedding, previous hidden state and the cell state which has been updated with information from the forget and input gates to update the value of the hidden state. - -```python -def fp_output_gate(concat, next_cs, parameters): - ot = sigmoid(np.dot(parameters['Wo'], concat) - + parameters['bo']) - next_hs = ot * np.tanh(next_cs) - return ot, next_hs -``` - -The following image summarizes each gate mechanism in the memory block of a LSTM network: ->Image has been modified from [this](https://link.springer.com/chapter/10.1007%2F978-3-030-14524-8_11) source - -![Diagram showing three sections of a memory block, labeled "Forget gate", "Input gate" and "Output gate". Each gate contains several subparts, representing the operations performed at that stage of the process.](_static/mem_block.png) - -### But how do you obtain sentiment from the LSTM's output? - -The hidden state you obtain from the output gate of the last memory block in a sequence is considered to be a representation of all the information contained in a sequence. To classify this information into various classes (2 in our case, positive and negative) we use a **Fully Connected layer** which firstly maps this information to a predefined output size (1 in our case). Then, an activation function such as the sigmoid converts this output to a value between 0 and 1. We'll consider values greater than 0.5 to be indicative of a positive sentiment. - -```python -def fp_fc_layer(last_hs, parameters): - z2 = (np.dot(parameters['W2'], last_hs) - + parameters['b2']) - a2 = sigmoid(z2) - return a2 -``` - -Now you will put all these functions together to summarize the **Forward Propagation** step in our model architecture: - -```python -def forward_prop(X_vec, parameters, input_dim): - - hidden_dim = parameters['Wf'].shape[0] - time_steps = len(X_vec) - - # Initialise hidden and cell state before passing to first time step - prev_hs = np.zeros((hidden_dim, 1)) - prev_cs = np.zeros(prev_hs.shape) - - # Store all the intermediate and final values here - caches = {'lstm_values': [], 'fc_values': []} - - # Hidden state from the last cell in the LSTM layer is calculated. - for t in range(time_steps): - # Retrieve word corresponding to current time step - x = X_vec[t] - # Retrieve the embedding for the word and reshape it to make the LSTM happy - xt = emb_matrix.get(x, rng.random(size=(input_dim, 1))) - xt = xt.reshape((input_dim, 1)) - - # Input to the gates is concatenated previous hidden state and current word embedding - concat = np.vstack((prev_hs, xt)) - - # Calculate output of the forget gate - ft = fp_forget_gate(concat, parameters) - - # Calculate output of the input gate - it, cmt = fp_input_gate(concat, parameters) - io = it * cmt - - # Update the cell state - next_cs = (ft * prev_cs) + io - - # Calculate output of the output gate - ot, next_hs = fp_output_gate(concat, next_cs, parameters) - - # store all the values used and calculated by - # the LSTM in a cache for backward propagation. - lstm_cache = { - "next_hs": next_hs, - "next_cs": next_cs, - "prev_hs": prev_hs, - "prev_cs": prev_cs, - "ft": ft, - "it" : it, - "cmt": cmt, - "ot": ot, - "xt": xt, - } - caches['lstm_values'].append(lstm_cache) - - # Pass the updated hidden state and cell state to the next time step - prev_hs = next_hs - prev_cs = next_cs - - # Pass the LSTM output through a fully connected layer to - # obtain probability of the sequence being positive - a2 = fp_fc_layer(next_hs, parameters) - - # store all the values used and calculated by the - # fully connected layer in a cache for backward propagation. - fc_cache = { - "a2" : a2, - "W2" : parameters['W2'] - } - caches['fc_values'].append(fc_cache) - return caches -``` - -### Backpropagation - -After each forward pass through the network, you will implement the `backpropagation through time` algorithm to accumulate gradients of each parameter over the time steps. Backpropagation through a LSTM is not as straightforward as through other common Deep Learning architectures, due to the special way its underlying layers interact. Nonetheless, the approach is largely the same; identifying dependencies and applying the chain rule. - - -Let's start with defining a function to initialize gradients of each parameter as arrays made up of zeros with same dimensions as the corresponding parameter. - -```python -# Initialise the gradients -def initialize_grads(parameters): - grads = {} - for param in parameters.keys(): - grads[f'd{param}'] = np.zeros((parameters[param].shape)) - return grads -``` - -Now, for each gate and the fully connected layer, we define a function to calculate the gradient of the loss with respect to the input passed and the parameters used. To understand the mathematics behind how the derivatives were calculated we suggest you to follow this helpful [blog](https://christinakouridi.github.io/posts/backprop-lstm/) by Christina Kouridi. - - -Define a function to calculate the gradients in the **Forget Gate**: - -```python -def bp_forget_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters): - # dft = dL/da2 * da2/dZ2 * dZ2/dh_prev * dh_prev/dc_prev * dc_prev/dft - dft = ((dc_prev * cache["prev_cs"] + cache["ot"] - * (1 - np.square(np.tanh(cache["next_cs"]))) - * cache["prev_cs"] * dh_prev) * cache["ft"] * (1 - cache["ft"])) - # dWf = dft * dft/dWf - gradients['dWf'] += np.dot(dft, concat.T) - # dbf = dft * dft/dbf - gradients['dbf'] += np.sum(dft, axis=1, keepdims=True) - # dh_f = dft * dft/dh_prev - dh_f = np.dot(parameters["Wf"][:, :hidden_dim].T, dft) - return dh_f, gradients -``` - -Define a function to calculate the gradients in the **Input Gate** and **Candidate Memory Gate**: - -```python -def bp_input_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters): - # dit = dL/da2 * da2/dZ2 * dZ2/dh_prev * dh_prev/dc_prev * dc_prev/dit - dit = ((dc_prev * cache["cmt"] + cache["ot"] - * (1 - np.square(np.tanh(cache["next_cs"]))) - * cache["cmt"] * dh_prev) * cache["it"] * (1 - cache["it"])) - # dcmt = dL/da2 * da2/dZ2 * dZ2/dh_prev * dh_prev/dc_prev * dc_prev/dcmt - dcmt = ((dc_prev * cache["it"] + cache["ot"] - * (1 - np.square(np.tanh(cache["next_cs"]))) - * cache["it"] * dh_prev) * (1 - np.square(cache["cmt"]))) - # dWi = dit * dit/dWi - gradients['dWi'] += np.dot(dit, concat.T) - # dWcm = dcmt * dcmt/dWcm - gradients['dWcm'] += np.dot(dcmt, concat.T) - # dbi = dit * dit/dbi - gradients['dbi'] += np.sum(dit, axis=1, keepdims=True) - # dWcm = dcmt * dcmt/dbcm - gradients['dbcm'] += np.sum(dcmt, axis=1, keepdims=True) - # dhi = dit * dit/dh_prev - dh_i = np.dot(parameters["Wi"][:, :hidden_dim].T, dit) - # dhcm = dcmt * dcmt/dh_prev - dh_cm = np.dot(parameters["Wcm"][:, :hidden_dim].T, dcmt) - return dh_i, dh_cm, gradients -``` - -Define a function to calculate the gradients for the **Output Gate**: - -```python -def bp_output_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters): - # dot = dL/da2 * da2/dZ2 * dZ2/dh_prev * dh_prev/dot - dot = (dh_prev * np.tanh(cache["next_cs"]) - * cache["ot"] * (1 - cache["ot"])) - # dWo = dot * dot/dWo - gradients['dWo'] += np.dot(dot, concat.T) - # dbo = dot * dot/dbo - gradients['dbo'] += np.sum(dot, axis=1, keepdims=True) - # dho = dot * dot/dho - dh_o = np.dot(parameters["Wo"][:, :hidden_dim].T, dot) - return dh_o, gradients -``` - -Define a function to calculate the gradients for the **Fully Connected Layer**: - -```python -def bp_fc_layer (target, caches, gradients): - # dZ2 = dL/da2 * da2/dZ2 - predicted = np.array(caches['fc_values'][0]['a2']) - target = np.array(target) - dZ2 = predicted - target - # dW2 = dL/da2 * da2/dZ2 * dZ2/dW2 - last_hs = caches['lstm_values'][-1]["next_hs"] - gradients['dW2'] = np.dot(dZ2, last_hs.T) - # db2 = dL/da2 * da2/dZ2 * dZ2/db2 - gradients['db2'] = np.sum(dZ2) - # dh_last = dZ2 * W2 - W2 = caches['fc_values'][0]["W2"] - dh_last = np.dot(W2.T, dZ2) - return dh_last, gradients -``` - -Put all these functions together to summarize the **Backpropagation** step for our model: - -```python -def backprop(y, caches, hidden_dim, input_dim, time_steps, parameters): - - # Initialize gradients - gradients = initialize_grads(parameters) - - # Calculate gradients for the fully connected layer - dh_last, gradients = bp_fc_layer(target, caches, gradients) - - # Initialize gradients w.r.t previous hidden state and previous cell state - dh_prev = dh_last - dc_prev = np.zeros((dh_prev.shape)) - - # loop back over the whole sequence - for t in reversed(range(time_steps)): - cache = caches['lstm_values'][t] - - # Input to the gates is concatenated previous hidden state and current word embedding - concat = np.concatenate((cache["prev_hs"], cache["xt"]), axis=0) - - # Compute gates related derivatives - # Calculate derivative w.r.t the input and parameters of forget gate - dh_f, gradients = bp_forget_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters) - - # Calculate derivative w.r.t the input and parameters of input gate - dh_i, dh_cm, gradients = bp_input_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters) - - # Calculate derivative w.r.t the input and parameters of output gate - dh_o, gradients = bp_output_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters) - - # Compute derivatives w.r.t prev. hidden state and the prev. cell state - dh_prev = dh_f + dh_i + dh_cm + dh_o - dc_prev = (dc_prev * cache["ft"] + cache["ot"] - * (1 - np.square(np.tanh(cache["next_cs"]))) - * cache["ft"] * dh_prev) - - return gradients -``` - -### Updating the Parameters - -We update the parameters through an optimization algorithm called [Adam](https://optimization.cbe.cornell.edu/index.php?title=Adam), which is an extension to stochastic gradient descent that has recently seen broader adoption for deep learning applications in computer vision and natural language processing. Specifically, the algorithm calculates an exponential moving average of the gradient and the squared gradient, and the parameters `beta1` and `beta2` control the decay rates of these moving averages. Adam has shown increased convergence and robustness over other gradient descent algorithms, and is often recommended as the default optimizer for training. - - -Define a function to initialise the moving averages for each parameter: - -```python -# initialise the moving averages -def initialise_mav(hidden_dim, input_dim, params): - v = {} - s = {} - # Initialize dictionaries v, s - for key in params: - v['d' + key] = np.zeros(params[key].shape) - s['d' + key] = np.zeros(params[key].shape) - # Return initialised moving averages - return v, s -``` - -Define a function to update the parameters: - -```python -# Update the parameters using Adam optimization -def update_parameters(parameters, gradients, v, s, - learning_rate=0.01, beta1=0.9, beta2=0.999): - for key in parameters: - # Moving average of the gradients - v['d' + key] = (beta1 * v['d' + key] - + (1 - beta1) * gradients['d' + key]) - - # Moving average of the squared gradients - s['d' + key] = (beta2 * s['d' + key] - + (1 - beta2) * (gradients['d' + key] ** 2)) - - # Update parameters - parameters[key] = (parameters[key] - learning_rate - * v['d' + key] / np.sqrt(s['d' + key] + 1e-8)) - # Return updated parameters and moving averages - return parameters, v, s -``` - -### Training the Network - - -You will start by initializing all the parameters and hyperparameters being used in your network: - -```python -hidden_dim = 64 -input_dim = emb_matrix['memory'].shape[0] -learning_rate = 0.001 -epochs = 10 -parameters = initialise_params(hidden_dim, - input_dim) -v, s = initialise_mav(hidden_dim, - input_dim, - parameters) -``` - -To optimize your deep learning network, you need to calculate a loss based on how well the model is doing on the training data. Loss value implies how poorly or well a model behaves after each iteration of optimization. - - -Define a function to calculate the loss using [negative log likelihood](http://d2l.ai/chapter_linear-networks/softmax-regression.html?highlight=negative%20log%20likelihood#log-likelihood): - -```python -def loss_f(A, Y): - # define value of epsilon to prevent zero division error inside a log - epsilon = 1e-5 - # Implement formula for negative log likelihood - loss = (- Y * np.log(A + epsilon) - - (1 - Y) * np.log(1 - A + epsilon)) - # Return loss - return np.squeeze(loss) -``` - -Set up the neural network's learning experiment with a training loop and start the training process. You will also evaluate the model's performance on the training dataset to see how well the model is *learning* and the testing dataset to see how well it is *generalizing*. ->Skip running this cell if you already have the trained parameters stored in a `npy` file. - -```python -# To store training losses -training_losses = [] -# To store testing losses -testing_losses = [] - -# This is a training loop. -# Run the learning experiment for a defined number of epochs (iterations). -for epoch in range(epochs): - ################# - # Training step # - ################# - train_j = [] - for sample, target in zip(X_train, y_train): - # split text sample into words/tokens - b = textproc.word_tokeniser(sample) - - # Forward propagation/forward pass: - caches = forward_prop(b, - parameters, - input_dim) - - # Backward propagation/backward pass: - gradients = backprop(target, - caches, - hidden_dim, - input_dim, - len(b), - parameters) - - # Update the weights and biases for the LSTM and fully connected layer - parameters, v, s = update_parameters(parameters, - gradients, - v, - s, - learning_rate=learning_rate, - beta1=0.999, - beta2=0.9) - - # Measure the training error (loss function) between the actual - # sentiment (the truth) and the prediction by the model. - y_pred = caches['fc_values'][0]['a2'][0][0] - loss = loss_f(y_pred, target) - # Store training set losses - train_j.append(loss) - - ################### - # Evaluation step # - ################### - test_j = [] - for sample, target in zip(X_test, y_test): - # split text sample into words/tokens - b = textproc.word_tokeniser(sample) - - # Forward propagation/forward pass: - caches = forward_prop(b, - parameters, - input_dim) - - # Measure the testing error (loss function) between the actual - # sentiment (the truth) and the prediction by the model. - y_pred = caches['fc_values'][0]['a2'][0][0] - loss = loss_f(y_pred, target) - - # Store testing set losses - test_j.append(loss) - - # Calculate average of training and testing losses for one epoch - mean_train_cost = np.mean(train_j) - mean_test_cost = np.mean(test_j) - training_losses.append(mean_train_cost) - testing_losses.append(mean_test_cost) - print('Epoch {} finished. \t Training Loss : {} \t Testing Loss : {}'. - format(epoch + 1, mean_train_cost, mean_test_cost)) - -# save the trained parameters to a npy file -np.save('tutorial-nlp-from-scratch/parameters.npy', parameters) -``` - -It is a good practice to plot the training and testing losses as the learning curves are often helpful in diagnosing the behavior of a Machine Learning model. - -```python -fig = plt.figure() -ax = fig.add_subplot(111) - -# plot the training loss -ax.plot(range(0, len(training_losses)), training_losses, label='training loss') -# plot the testing loss -ax.plot(range(0, len(testing_losses)), testing_losses, label='testing loss') - -# set the x and y labels -ax.set_xlabel("epochs") -ax.set_ylabel("loss") - -plt.legend(title='labels', bbox_to_anchor=(1.0, 1), loc='upper left') -plt.show() -``` - -### Sentiment Analysis on the Speech Data - - -Once your model is trained, you can use the updated parameters to start making our predictions. You can break each speech into paragraphs of uniform size before passing them to the Deep Learning model and predicting the sentiment of each paragraph. - -```python -# To store predicted sentiments -predictions = {} - -# define the length of a paragraph -para_len = 100 - -# Retrieve trained values of the parameters -if os.path.isfile('tutorial-nlp-from-scratch/parameters.npy'): - parameters = np.load('tutorial-nlp-from-scratch/parameters.npy', allow_pickle=True).item() - -# This is the prediction loop. -for index, text in enumerate(X_pred): - # split each speech into paragraphs - paras = textproc.text_to_paras(text, para_len) - # To store the network outputs - preds = [] - - for para in paras: - # split text sample into words/tokens - para_tokens = textproc.word_tokeniser(para) - # Forward Propagation - caches = forward_prop(para_tokens, - parameters, - input_dim) - - # Retrieve the output of the fully connected layer - sent_prob = caches['fc_values'][0]['a2'][0][0] - preds.append(sent_prob) - - threshold = 0.5 - preds = np.array(preds) - # Mark all predictions > threshold as positive and < threshold as negative - pos_indices = np.where(preds > threshold) # indices where output > 0.5 - neg_indices = np.where(preds < threshold) # indices where output < 0.5 - # Store predictions and corresponding piece of text - predictions[speakers[index]] = {'pos_paras': paras[pos_indices[0]], - 'neg_paras': paras[neg_indices[0]]} -``` - -Visualizing the sentiment predictions: - -```python -x_axis = [] -data = {'positive sentiment': [], 'negative sentiment': []} -for speaker in predictions: - # The speakers will be used to label the x-axis in our plot - x_axis.append(speaker) - # number of paras with positive sentiment - no_pos_paras = len(predictions[speaker]['pos_paras']) - # number of paras with negative sentiment - no_neg_paras = len(predictions[speaker]['neg_paras']) - # Obtain percentage of paragraphs with positive predicted sentiment - pos_perc = no_pos_paras / (no_pos_paras + no_neg_paras) - # Store positive and negative percentages - data['positive sentiment'].append(pos_perc*100) - data['negative sentiment'].append(100*(1-pos_perc)) - -index = pd.Index(x_axis, name='speaker') -df = pd.DataFrame(data, index=index) -ax = df.plot(kind='bar', stacked=True) -ax.set_ylabel('percentage') -ax.legend(title='labels', bbox_to_anchor=(1, 1), loc='upper left') -plt.show() -``` - -In the plot above, you're shown what percentages of each speech are expected to carry a positive and negative sentiment. Since this implementation prioritized simplicity and clarity over performance, we cannot expect these results to be very accurate. Moreover, while making the sentiment predictions for one paragraph we did not use the neighboring paragraphs for context which would have led to more accurate predictions. We encourage the reader to play around with the model and make some tweaks suggested in `Next Steps` and observe how the model performance changes. - - -## Looking at our Neural Network from an ethical perspective - - -It's crucial to understand that accurately identifying a text's sentiment is not easy primarily because of the complex ways in which humans express sentiment, using irony, sarcasm, humor, or, in social media, abbreviation. Moreover neatly placing text into two categories: 'positive' and 'negative' can be problematic because it is being done without any context. Words or abbreviations can convey very different sentiments depending on age and location, none of which we took into account while building our model. - -Along with data, there are also growing concerns that data processing algorithms are influencing policy and daily lives in ways that are not transparent and introduce biases. Certain biases such as the [Inductive Bias](https://bit.ly/2WtTKIe) are essential to help a Machine Learning model generalize better, for example the LSTM we built earlier is biased towards preserving contextual information over long sequences which makes it so suitable for processing sequential data. The problem arises when [societal biases](https://hbr.org/2019/10/what-do-we-do-about-the-biases-in-ai) creep into algorithmic predictions. Optimizing Machine Learning algorithms via methods like [hyperparameter tuning](https://en.wikipedia.org/wiki/Hyperparameter_optimization) can then further amplify these biases by learning every bit of information in the data. - - -There are also cases where bias is only in the output and not the inputs (data, algorithm). For example, in sentiment analysis [accuracy tends to be higher on female-authored texts than on male-authored ones]( https://doi.org/10.3390/electronics9020374). End users of sentiment analysis should be aware that its small gender biases can affect the conclusions drawn from it and apply correction factors when necessary. Hence, it is important that demands for algorithmic accountability should include the ability to test the outputs of a system, including the ability to drill down into different user groups by gender, ethnicity and other characteristics, to identify, and hopefully suggest corrections for, system output biases. - - -## Next Steps - - -You have learned how to build and train a simple Long Short Term Memory network from scratch using just NumPy to perform sentiment analysis. - -To further enhance and optimize your neural network model, you can consider one or a mixture of the following: - -- Alter the architecture by introducing multiple LSTM layers to make the network deeper. -- Use a higher epoch size to train longer and add more regularization techniques, such as early stopping, to prevent overfitting. -- Introduce a validation set for an unbiased evaluation of the model fit. -- Apply batch normalization for faster and more stable training. -- Tune other parameters, such as the learning rate and hidden layer size. -- Initialize weights using [Xavier Initialization](https://d2l.ai/chapter_multilayer-perceptrons/numerical-stability-and-init.html#xavier-initialization) to prevent vanishing/exploding gradients instead of initializing them randomly. -- Replace LSTM with a [Bidirectional LSTM](https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks) to use both left and right context for predicting sentiment. - -Nowadays, LSTMs have been replaced by the [Transformer](https://jalammar.github.io/illustrated-transformer/) which uses [Attention](https://jalammar.github.io/visualizing-neural-machine-translation-mechanics-of-seq2seq-models-with-attention/) to tackle all the problems that plague an LSTM such as lack of [transfer learning](https://en.wikipedia.org/wiki/Transfer_learning), lack of [parallel training](https://web.stanford.edu/~rezab/classes/cme323/S16/projects_reports/hedge_usmani.pdf), and a long gradient chain for lengthy sequences. - -Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as PyTorch, JAX or TensorFlow — that provide NumPy-like APIs, have built-in automatic differentiation and GPU support, and are designed for high-performance numerical computing and machine learning. - -Finally, to know more about how ethics come into play when developing a machine learning model, you can refer to the following resources: -- [Data ethics resources](https://www.turing.ac.uk/research/data-ethics) by the Turing Institute -- Considering how artificial intelligence shifts power, an [article](https://www.nature.com/articles/d41586-020-02003-2) and [talk](https://slideslive.com/38923453/the-values-of-machine-learning) by Pratyusha Kalluri -- More ethics resources on [this blog post](https://www.fast.ai/2018/09/24/ai-ethics-resources/) by Rachel Thomas and the [Radical AI podcast](https://www.radicalai.org/) diff --git a/content/tutorial-nlp-from-scratch/README.md b/content/tutorial-nlp-from-scratch/README.md deleted file mode 100644 index 67bffe7b..00000000 --- a/content/tutorial-nlp-from-scratch/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Data used for building the [NLP from scratch tutorial](https://github.com/Dbhasin1/numpy-tutorials/blob/ethics-tutorial/content/tutorial-nlp-from-scratch.md) - -## [IMDb Reviews Dataset](https://github.com/Dbhasin1/numpy-tutorials/blob/ethics-tutorial/content/tutorial-nlp-from-scratch/IMDB%20Dataset.csv) - -**Purpose**: Training the Deep Learning model - -> Information courtesy of -IMDb -(http://www.imdb.com). -Used with permission. - -IMDB Reviews Dataset is a large movie review dataset collected and prepared by -Andrew L. Maas from the popular movie rating service, IMDB. The IMDB Reviews -dataset is used for binary sentiment classification, whether a review is -positive or negative. It contains 25,000 movie reviews for training and 25,000 -for testing. All these 50,000 reviews are labeled data that may be used for -supervised deep learning. For ease of reproducibility, we'll be sourcing the -data from [Zenodo](https://zenodo.org/record/4117827#.YVQZ_EZBy3Ihttps://zenodo.org/record/4117827#.YVQZ_EZBy3I). - -> Andrea Esuli, Alejandro Moreo, & Fabrizio Sebastiani. (2020). Sentiment -Quantification Datasets [Data set]. Zenodo. -https://doi.org/10.5281/zenodo.4117827 - ---- - -## [Glove Embeddings](https://github.com/Dbhasin1/numpy-tutorials/blob/ethics-tutorial/content/tutorial-nlp-from-scratch/glove.6B.50d.txt) - -**Purpose**: To represent text data in machine-readable i.e numeric format -> Jeffrey Pennington, Richard Socher, and Christopher D. Manning. 2014. -[GloVe: Global Vectors for Word Representation](https://nlp.stanford.edu/pubs/glove.pdf) - -GloVe is an unsupervised algorithm developed for generating word embeddings by -generating global word-word co-occurence matrix from a corpus. You can download -the zipped files containing the embeddings from -https://nlp.stanford.edu/projects/glove/. -Here you can choose any of the four options for different sizes or training -datasets, we opted for the least resource-heavy file with 50 dimensional -representations for each word. - ---- - -## [Speech Dataset](https://github.com/Dbhasin1/numpy-tutorials/blob/ethics-tutorial/content/tutorial-nlp-from-scratch/speeches.csv) - -**Purpose**: The trained Deep Learning Model will perform sentiment analysis on -this data -> Curated by the authors of the tutorial - -We have chosen speeches by activists around the globe talking about issues like -climate change, feminism, lgbtqa+ rights and racism. These were sourced from -newspapers, the official website of the United Nations and the archives of -established universities as cited in the table below. A CSV file was created -containing the transcribed speeches, their speaker and the source the speeches -were obtained from. -We made sure to include different demographics in our data and included a range -of different topics, most of which focus on social and/or ethical issues. The -dataset is subjected to the CC0 Creative Common License, which means that is -free for the public to use and there are no copyrights reserved. - -| Speech | Speaker | Source | -|--------------------------------------------------|-------------------------|------------------------------------------------------------| -| Barnard College Commencement | Leymah Gbowee | [Barnard College](https://barnard.edu/news/transcript-speech-nobel-peace-prize-winner-leymah-gbowee) | -| UN Speech on youth Education | Malala Yousafzai | [The Guardian](https://www.theguardian.com/commentisfree/2013/jul/12/malala-yousafzai-united-nations-education-speech-text) | -| Remarks in the UNGA on racial discrimination | Linda Thomas Greenfield | [United States mission to the United Nation](https://usun.usmission.gov/remarks-by-ambassador-linda-thomas-greenfield-at-a-un-general-assembly-commemorative-meeting-for-intl-day-for-the-elimination-of-racial-discrimination/) | -| How Dare You | Greta Thunberg | [NBC](https://www.nbcnews.com/news/world/read-greta-thunberg-s-full-speech-united-nations-climate-action-n1057861) | -| The speech that silenced the world for 5 minutes | Severn Suzuki | [Earth Charter](https://earthcharter.org/new-voices-after-26-years-of-the-girl-who-silenced-the-world-for-5-minutes/) | -| The Hope Speech | Harvey Milk | [Museum of Fine Arts, Boston](https://www.mfa.org/exhibitions/amalia-pica/transcript-harvey-milks-the-hope-speech) | -| Speech at the time to Thrive Conference | Ellen Page | [Huffpost](https://www.huffpost.com/entry/time-to-thrive_b_4794251) | -| I have a dream | Martin Luther King | [Marshall University](https://www.marshall.edu/onemarshallu/i-have-a-dream/) | diff --git a/content/tutorial-nlp-from-scratch/dl_architectures.jpg b/content/tutorial-nlp-from-scratch/dl_architectures.jpg deleted file mode 100644 index f9c07677b9b53aa8400222650e90e6e43367453c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25958 zcmeHv2UHZ=tMHC4lAQ?e2 zNCwHd`47&Wxx&nw@!fUrdhfsQGOMON-BqW~Is4oDd^=Pf_8*Qyrxax5Wgrw(2ton> zL5G8o6oie5iG_)QjfI7UgM*EWPfCD~hlfu^a+-*gmYVJ?Ej0}dJrmajdIksTv$6f#X4;}^t1|cFMpF~B+Jbf0M zl$?^9mY$KBRajJ9Qd(ACQTg^={riT-rsfZwUEMvsef^&Y#>OWmr@l_l%+9T>uB~ru zZf)=E9?c5{Li=S};Qzl&?8Ll?!Msq>(b3Scj^>4e>IM!pVswmiJeVYx)v#_lo@U_n z#U{NHo>)+a!^o%pjm*TU1DBkMe~fwMXlmbQ_Hz^S`-f)s+r<7puK@@j4Fw1fjTnMK zTe~btKDhrm{<8-E!N7ko@E;8P&tQP>IH)*z`GNonKtv|G388RI;HhACsL!!u}b60O8XjC;Ze;BC)CjrSkoL?8 z(SjT*+auMvq&f21q(Fw0+(`n{=6z@~*-AL7JZS-qxD)=xv~==?r>y4^>1XCRx$uq4p z4BlAkK1pt`}lEHL4)IkA*o>!ZK$iX^lwL zBr*Q49OmwB$ji_nba#BLR^&x*uDQE2Qc`<*ua(i;zQISpDhUCx{5u~x&EnFD^;1l} zcP1U{v@0cL=amkj`_Cm0p@gr`4CiL&7HLJdPSXYd#synRMv~Z}A^l?= zkQj|few1u!bNE}O1o$UsaP6D&L#QEaTfUfRm(aR)n}Tlp6DC+y2c&ch+QE#!_q~oo z2uINO5W3tdbO@nN975Es-?jpa8{R>7%7WKO!Raj>9&*8S3dpI~zEK{26G)`Kfw*EWm#7+0qM}xWpB=@qj z6G;>3s*{B7dErPBWi;gu)U62Zw$dSM!;pcok^LF>3+uR$uzW3krNli_EKt9Lj^c(XCb}_pojWc#0G{K7mJ(B1ovf`n#0y*M* ziN0-7HmwJn!F*r>vhlsKAs?spq(3u?CP*@_5QZ~A`AqGnU(poq@Cp)@YN?E=+lYE} zFh8{QV6vq$BNkWZaXu}Cs~LA&83n7`l(c;()T(x4^AOrq;l$Ak)zC=$+}j~dme@e5 zCJp^N9Ay6E4jZws1329Qv4Dd%5dyQPa|-4p)=|D|kGM*=y~eU{ErAHA58is(ZzEV4 z+TcD;L|P{!7qSwK_8K>H*P=4R67iKZ7KcRcg}D)-6p^Xnl%}X%>85-AD?XZnvS;I8 zyp-SaZFjTk&SuI!8=`)FzAAWjIW-xxqDF(fuE$hdEM5}RI~_HvTX6HS{?f$;vKLeR zFN)}#JYXS83d&O&=}mz*s+2D1?U_y6rPg1eN?D{*SKwJUe&#DtC}vUl{>BF?IX0Qg zv;@vfs261}XPGLNUJI<=Cf$(ZVhU2($I!*K(Pl`cxNyhDJ$h5bj;Ukp#+Hy`p)}2o z+F7dSUBSL{aO!+DgMm7m2lPRv=#z`IDw!JsoadLUmE##0{1*9hO&25Yl@`^Xi&gH3 zVoIjl^tM3XY+8(+>+xqXr_ZE9za#K5JlwxCGlciE6T{l5Mh|^?1T*X#*$viA-L|7U zTe~Bc#Gc*g9KW&Zktx1?2t`{3I8A?fWk_@_u7^O{!Hyuu+mM*UMp;g+dpBM~z0>~d zld+~8p=&i^iaqv>6*p;X=GhzBZzriHDnt&R(>a?$JZq%n)E30XE0;C%4nxltuSX6c`Kfd%bnO-Fgb>EyJDQZJv9+*3{yW$8EF;tCmZs3|n%dAMDKk5R#Q32ltGKGN;mgl)?hgKazJU=0gwySZ~L zg@pgWDX;EvJQ^1>sGuF86PN_+gLjR73woX9pB-MojZzK5Z)u?Q} z>t_$4#~O!F-%uYiEaLwF+5LgYv$MY&S;c{EY&ai6$-^bK?C)wwwXc zs-pZ~IrOg60vre*LV0V4(9CqM{)cbN`@WILDiyFa*Z=zF4?Q`rd!*uv zcUL;81$c`z?tHBjMO9O@yq$z#VfxFBDreDI)s9;>ZWiKYLE-CS8ZSI8)x%T}y2@!W zy|_TOS%sXBa|~NsPdAl&JGa}iuG?wHtyOi)R$oQAHW;O>xQh49Noyt&*w|LE>*^zxa{0r7>_p-f|*jSRn zpZ=x2)TKU{>ZmE`%=vmz{!V>$a#qIr0i4aii%zX;gVL06yu zZis7FEOh|g27_o#0_vY0*%?Cz{z~PQ4HG2fXx*E6u6^>|C7e70c-RN@A|FzUFRFMc_r8J$?SG`^%R*7e3a+BaDF^EzMq7y?uU z8*r!v!q0isozWUhXfy_Hx7@AT>4#u5^=pYo3K<@}|InPFkcqMN`9ekL*o)adu^4;a z^3|O-(Jp>v-XYyZtDOAfmuR|#QR-XYH7pCVhO-7|d}e;qfr<0G3x zShFKeNWjwhjnRY44C%RZDZ=TlI#B&IxCY=54R zDnJhKWQ}`kcQ1*K*m<|oK6#fhNkd~@hVFe+B*#W7s=t%B62xW>p+zM&PsuvHH*Ads zSd2T`7Y6pP-a4%kro>~IfwP%^(GGIUH9js%*?lHKG!JUWZm$=_KUI)p>K~r z!`lgf05D=;`}W~ny-L(2&%N^ct+(LjBNUFXWjQmHDOxF z1B-D?xW7Y0*(P2^%ug1;;~{5b6BjsYoCP4<=1y8+(iW6}iy zuph9l#m2IUSN@LyHAi>8l=?y6XS-oX!FTj}b}5h8u+c;2`260jF@y0;y*Oj>*f-um;4)x7V{h068&`v-5(oGo#x2%*sNYioKZ5Sn<}o+;{8h4VQH zy|8Ez{=qOBeS`T9>$G>OE)mR*^u>=aG6#MYRP7&i;_DlTWN=mg)Bz{8w&n<5uCxdl z5ajuMZN5St-Rv9|I`MmaaqE5phy%%|}|9XMd^V2YyT3 z|7-2VPlEROrcm#0NYapg75_E8%VuUG>Y9F(sMz$O(zXCE0E5l`wX zUOa@tf<&Kagf$ey#?BQULUGhFNIWk1_K1X>Wu`7$v$Iv>)-}f(6}9-a z-ekXN!)_NbSHax$&+j8yT=C5|XSENZClwWKQeiY`@BS6-Pyz*43$ciHt?g#n95pWADI5I|KDK|L-)S&s#gvan$AUhm# zE|i%1vXQl?*Eru+8OI~wx3*5Bct+6@I%Rf7asiIpcT!yN6V&B?tLY^zi#(39_M{nW z!DQ}uQLZyqP}NAM^W4CgfVn zTz_7Wt~RXCe*^tf86WpwZ9bJ?zcBruAQ3-l6#fe0f3%aOz{zKcWjf`y0ugw80{2HD z(FoVuY%eY6Ygf5#;2X2tNFQra9L|O;zA_M1d(nNwlfQFdlv0o#0{)`Kk&^^pg-UjX zkoT?7wEvW-vn1ftordRh976LT&Nvg4RigJpvLRbXi|Bqs+L$z{b^{2+{LTZ%P?9<@ z%0GbXgG;yP3s3N;h3x3sdOMXHV3?;Rk6fKRW5BP%^X8G&b|5e&GM{ns3k759^Kk59 zHECBGaL?b}4n;ow5NhfJoaDL?-7XDjaV&g4#I<%~0`S|!I=!B4s@vP4;6{0TM{eAa zcZC+xbqMtg!TUCj0xG#ZO0WZ?z5}=sd}fDj$W3ZYmcI}LlIh+ADdboD-}A`t zS9JD$0Q+158n3{Zn{~XMR(85SRhm@Cf&f!W?Ji0#Zu^h->hvVA zcU@PLzD;}XXzlf`p8>+U+kOC6Fp;_^C!YXZTV;-%-Z;uH{(*3Q_zsgOsJ0g*1~>NM z>sNd9bGDx!ScCyu4VeE`C9UkNfVZX^Kt$}OfAfM722bg46t>cs1^ax&;D6rFr6>*g zp$f5oXy|}RfyH&=!$dW^-W2MjB&^{5F*N3dlg|FXp`K-eU4;+XC($GHddo&8e z*NKM^BM`LvAONOwpW2BpdY`=JzlJ2XN_FpK@sA*V92{9%Ztsm( z(LhYC%J_gKG*d^3;8~Os7;x@KzwfKSCSv&z!Y@p@vHqpM6~_>VB`rB8kd~4+?mmsb z@^2nL{(&1(dQk;g!wh_E0@sat+rL{T+p;{Hy<{6tux7f{-xtngI~yqp-`oO9n@Zf< zbFA5A5VW8UU{L+_JG75V#n@Qw&-tIdlX>>8{S7{KcJ_?#856g3%W^~gUuE&5DYU&( zv@xmS086G9&{k(^B!qc?46Hi_YYvS zIZ$%0v`;D}mP_kzG82E=)s~{B9aDfa8SreN8u|0a2XwTqS}n%o zg6Xf+q{)I3j+Q}(I|JU-2Y`275M(B4Ho8xKk;)x%8g?Hna24Tq%bCH=g^`<5L^pO_ zqTMF$mAJg~>U*2?{I-#<-QFjO?fgM6DomZvk-L+#A3}h!EXl(1VNtG*3Po2U5_b$y z*`s3Ab$VGiQU}fMwYE?W})XH770*uG2bP?zT(C2%yIt0<=~|@FEO1& zT)WFx1|-_#m7JYj#cS&+N zg*$O_q&_dF=^IXKd~dGY*K$&oJm5Aie5P~h=tVxV3NHr41<0wz8{D@n0`Ea=GAia$=kTvWHO+1LZH z-m*ACtohu4S3`n{?qrDRwaD@~l_`B5nOASaFAPcJvaV~1l#!hcRA3c*CYlz-IrY(b zu#z#Ojx%nEz$S0;QSF(6Hc^Xc1kIrQC7+~7J7AF2kFfSr#zW2Ui>$T%+6TF-^S*K0 zX^I~Bg2uK^_i_`Q~6k?S?wp`pC z&0O0;*Du|i7^A%%-}C9S(|;~M&d2RB0;5sG9pO>tL*8|OfK*UHZ(Qz z6MifJyn*p2$s%hc4mB3>DJHfnk)No&Na^|bLpX~MA%RqL%X^T{tL&KQ@wrZ2qLn<9 zb&0IIY*Qj4&JxXP^15xExAHj7xu@MRzu>fZAxys`385YtSw*d_eOa1RGx!LfKf^h7 z;#S!`#vWdA`zUvfXl?##-{ZkAQh1TCO|INfXZvL${|#HhVpAG9Jjijel4U6Ri;93m z$g85FP(mEFX4>|<1=2cVQ&TfsTAt4;Uq2o2lQA{dJGJ4UP<1P!(T@6>lw7u-PGY82 zVrzNED>eBqU;^QQ&`ACqqh*a$cy(phXsV43A0uVqzTL(%)-tVh5o=!>-d+sP6H_}^d zl?B4~pHd>V&9qtB?KGpcawX}{`Efik%L%EJS-Et%aXP1fqT8tN`B4Ajhn_a{#zbeCVTYJ zwrNfiV2KTL2iI(k7 znJH2=xu*Y)VLCESKOUOhu%07IypfL#3$0?Ca3R_NzS#jFe#sMgA1ZA3fgi(HGO~mE z#S060pPg;d1-3xb)H`Ca+V!Aq!7L`}sA6uc z7x?f8X3x5CGE%DF13DLm=de3j+oZV>lfl$GYf*4boiu`fK(tJFa@VAU^;Dng&)6cq z__v&-YDNLVN0#K|@$;5tngcVzL4c^Bq}*7d%%fsHvWv`8a^$P=&hvZi!b^i);?rcQ zcIITsA6KG3w$~JT_IPk4mXuYvlt-YjdqiCFpjiu3a1(S^ujFUR64i}aSn8w_FLg2@ zdGSj5b9Q%dY*E2q5}fG26PzCq4AzuW({?bfRz0^zKAXsuy3PHtAl<7Z z;eyBDl;xmEtDTBD+57s_6?Weubg;r2SCc$h^0??QsJMNtR_cgmBq-G388{bZbj*p(((013_{AY~rvDZ2 zKYXGc-?WWt9I21H%(1U$2IAfuDIzZe2dlX@>Sldq{3Z(95wr~aUk-Y-m)a)BGRNVZ zySM1V<~U=+To}Z0sR7T!0M}(2vp|%QKPvE!!w$LuyyC!S!^Flp@1;pJ`Rdcb7A6sK zA2A{)_gn*c^wQgvrK76O^O7mA`IsKqt#oe>5I{=)l~V zyE3|s=v!#u;f}a%)&_gjS(a7x`RZ23FzxUy+hXSi5l}5ODr?64;o}_&)B4idTRc?a zX+Y&o3FeJ3bUaz&cE#Qrw1&8bx;Mfm^m+-z-8vqnJt-dhj2(aX_FdD2@uwp-`KA_y z)qH^9ltuy;=Lnx+IN~_4Rkm5WjVU6e4=Yt{@d*_tqgg3_C2r04_TxpSU{>?3Grn|B z1(-=uq7EWG47Gq*wzk>gba449?pVlf$0Jv9CXiOw zEoWpq7XAry4~YchS{ay0WlGL`GCuez77>6vAHRVM^em)8KHbLnJkVEbAV(lSBo58ir%|XAmRBYRyG@MzH>Q7~X3!BQ%B=wZrBGfiI zd3y4RVOXCX0}0~{_&DG>l4q9p_8bnOMi5x}@fOJgvgJ;NHf(gq#tb`t$!7QxP0phe z=}oScmrreX5q?gjvJSqBAGQ2BOEXn zys?3U#*ujl{SFl$ZLgp0&8II5ZLBZrsBEJezD(`&R6@L1y^-m*GE5=M?rTZk*N5Xv ziWS;#NA7g8Wje&!e}KS9Ug{4V+`G)#^ZKPkH?RJCN~<`YhdPfROI2<<>L{f$edg1- zq6Q{g--YxM8rj+e)rz5DTBlWZp%M7VARH-XbiGsJqlcaV8PUEHYzEEi5Q2l4KXlgxn=BW;Q@X>J zf(q~53tJ`7{YM4%Q;^I0R>#~JWM_()v$fbGt6MvVZE#>FesAvGdOG~a1?b*YD%soi zW6ur|_0%;jiRSIBw&ZB1g3w(T9ww!Xk9V41;pCJ3bJ0HO|A1_+0Rh1Pkj1Wb9oe@@ zF8^fDmBgczZ=-yFG9(%(k4>pn0=|J4s;3q^VNE8>1*ThNur{_tSb$5jz;>R=io~2= zib77#sUpFzO#`RsM?>W;<~FN!x2$&z{3 z*y_7<&&%kw>9$Gfykfm;AH`ihSzA0MwpMgM0dovLh2Yz|zAsqkZ0U}hzOrPd$ciUj zo~Qp@j}3r8`KHYNT zcf1s?>L?Lzg!{0)HE;r@h_%Jb$4u-WTl}BG(jS`eyyGF2O6&E(OiPc`2DJ}#CuON6 zwY(nAH19X(S*n-zEZ?cB(p_hNvnIKcX?^1jvmDjIw6bkk4!gWyjxBm;_em`cTX;UQ zOXjXazGx?IHH+;tt)`T+|)Ude!W40y*d+R?PPk^%2mt zKnRvjW#`nxmU_4O+wWbAP z2}0Y6u+7ZLCmu93nrWrY;pER2C&BJ{LXN}6Q)hyCJ3TY44AY|H&rNh(r$n2#_Rm@0 zKkZ4jIP5mSh?ZDY)gYco-cqqa|L6n7s%P}0L@7~hr71Icm1Ohm_1#FMQJZnySpk^t?&GzgA_k4bC>gZ^ER88Ob;X7X=GtbxyP&g;;T>?Df zvV;(*_F09m%w1~F2TG@+1d536#qZxOvC89fW?$GngbvWxNA@Y20YAv9rdH0iV~t=R z;?!MH^Ztl5IkLp=9)aokNWqK#aHa_)^D>CHXnq5j&($xpHm?qag#o%tJw z&?S&syS4SGe8=FDDRspyvyC;VaqrT;i%V1rym7PG%OL<6g!r$*f5)FpWG8CSUUnW` z)MJeKU|wo1WGaHUM!fAC)=-5FPthPfn)!1P zo2X;tryDMdIx|1_CJm8x+J421?JK*&OZOfi_{8JSFJGkREX!z+qgm@pwVCN9nT?e$ z3GQQ<8ddUBea?%e%zGD~uqKP>G`uN4E6hM13n^-={m*$&{lf~;zW>dmTm%dxnSe`i zT=;(&-T1-T`!(W$#-_4Ey;zfD4$wXUDxl#z)MJ|k@B;$&pB(20K=>x+buX2Hq@%Pf zh_WQZ^LkWr*Wfcaf1Un0LI_#ZxWf*aK-lgI$o)wQfa?77AR(#&$Xq@Z05aOo^>Wmc zxqA#!`la06!SVxk?ZRrnTo|UdHv$nozva|Wf6GGtRb}lT{9afM_$hrqe15n6drnyY zxCnj@tw+(v53p5%{E{HB;*Hy&l$L0Ry1gV2v|4RGq5P2D{V)Hmr_Vjy-JZK1-)yU0 zSbQngcjI-;4g&fqk>}(CEN14w`+`A=3Izy!MQ9%p+j!s(=9EDFbA139p|RJ;uW}EC zX@NB7Su!9qMo>vl@MePrwz83{|4(^x7-;X0<_u)UPl3$1Al)vq=qS1|^OqGDzX}^4 zxXa;r#C@kjQ&}r=5g&$ypXQi!RD~qmP4tby! zS`{<*SNr^DpXTn*ke^?jY0l6&O> zkXQA=k`Tk2XajP-5$z+Qvj6dC*U5mFp$qeU2cN7n9_3Ot?h(o9-_oyIA zKyJ`u4z|q+a)aY6Xx9-4MXDDe=lHW0_nu*xMWa;JP4|geFSE5z@yN> z$J?3CID|$^CqQK|s2x4g(Fu)819eD8r6=tYAk>Oc1}dRddj(n)vsoq{(n`NDGLVzq zba}I*Ze}5#egh|gF!rA|j-Tr&YUTWkPMLtkKr@$+Yo&ZiOD1HOEsiRhBZo^eezm9I zv%R`^A;o; zHec5U<*cCUX003k&A<^j*x)yuqoj-Y2HigS7f=h1lK(Ru5k3M*-F90_*t&?VcE!_@ z7U>m?p)RS6!0Wx(9koSyBIX{7__x|+jN1U)IWa8AwnOiCLpw1nRm>W)@RLJ3Iln*b zNaORot4^P>`TZ{(@9}*tN}@S3=t+b!+VHql^r273HmX8{`dFuC2BK1tqsrL+Hc%O> zjS__|Odl%23LE=}fcG;3pJxYo=`i-lc`p|bT912vVwz-9vVEMKx2EJ1yR`=9#wB4z zc|;$}6~{J_Ke-Tx#3{u~l?3TymTOAMY{SOh&{;RwD@W|DI1&8f@iPGVX=aHXuR zvRBrdxz0`>3s|+7)m}t=B!x&x1+_3tA zJAxnM*@$vEisYO;>zmxzA2zk@;o-XUfs)o1-uhI{C>bZki>i>SJkw)g+8NaM|DZzr zLz*Q9wn!Ycd6!$TRZ;C+XVJC_Yd4c{DZ->SF1s&b3-i9KIZYoul;51z0R_zeGEMmd zTc6fE9>PmGtS7xH<5MIy+Xu1G$}F~@6c$@dS#%jYp8WfD)7ES_q*KA==XhuIyzJ`_`EWg+v!Kc+FpXaT|-;$ z%j3`-Npv6QV9?D`p<8%fBeL2azAnr*r2%OC0(`z<8srE3e1xn3%{-!-N5^!r!}c23 zy7T92L3K=&v(yx>T58<|Z%F4Lmb9vr8x-cMa#R;+ zB7I<@vau{rIUT+&@}V)|@$iuDZ+gW-pwSq5T<@?xS_ZRP)CVf9%{O z%(!+mS5g$BquMYMP?fji25Qqtv}?@3J5erkgT;-3@2WA|{>FJRWrJXoB=H+NU^h%9 z?(QKv)&x@fo<8Ikr7*)!?MBrR_qhvalq5_-*i`cIX0x)~eseZ}s%{?}-1SpS z^YIOuY_vKmtd(?d4*?^+r|hVjV?GjueC?ysqCU8+8S@P6$(S-!+8U2#GtyNtEdy&O zb$e>(+Z;eesJ!9GvF6{|m6>}a@-izJfAkSJY0lSDGbBPr0ADdRqH?rrJP|BDL#i`fPKfPj0*+AtdymA=Vo{_M0Rv^ADu)uVM_81Izo~R`DDA zBfBaBx>=x3_V29()bgNZ0T8jjHhEvX8Fi@PRRi%y2YXZX#=y;uku?hRQQ_qk zR~?B&g6+;~V1znBLsb|2gNc-qe8e{gwsaYIT--9!p4d}TjXso(g1cfH7GFk?w~O7~ zuV^HDZhJQU;8BY1P>8zFqdo40&E{K!v{%E6)2?yjW8az#N!00!*F5Uz%eMc*>PP0< z`vdQ8tw=t8AWxgeOoA%7wu;fTKf0|ty zcd2Ui64y2<^GNnN1200CHG52c@O6}tehZ7EN(Q@8V^B@@uF6Yij;vjlAg7uF7(=T^&{l zL?VT?efp$((!GfXab1#JX}V^7S8)}L(d$#p>*;;ctuBNQ3Yv9?J+xm)wYzEJ=`q|v z(otrp(8ssDCaH|0!ZJGX&_vX6o>UbpZV3KX@@7xmu)&_Q9mc`?N-1nqd2P2<-!tB~ z@`RD}o*f=pb1SwyOl~^kn+Ee456T{Uw{_%Jx03e``Z;ANpAq>&q%47((U?5jVZ)gr zYmPfq!1RIQMYx}L3ZcqfsJXK~S-Q|x#Nr}zNYT4uGdX3q(~V5Qmh-Vs+cUp9J>UPt zv?CaSNz@V+Lo4d!Y@k|4R(yrURCfaXan?mlWq7W5(bOG>RK@m zdi)7!my;A_`m}t`uopr?UFqr&UX8%SLw4;`&6Mu3viG<@NMT)IvKy#|Jn{mx%BzYh zBli?+S1ia^&X-&=I@!c>)bB$f!l~JbQdy06q(da&`!IW>8a9$>m z5A|WjTWzV@P~?^Af3VgVDLPYdtHF|C;FSN8S=#7?@MH%1i?x|p9}V{Rr|W91k^$P{I-! zX^4AkxH46eXZv(IhTF~YV|W$^8M#zGIMo`G$gNATpyA0z-%RIgVQN}-aqlWttf+qf zG|Rtkl;zE$Ps!bZ@ukaXd>_qC343r>dC80>Uzm!QdB0ofvb)sr`E%~Xd70s9I`>ih z_%|rqT93i2u6;ytTkFa^ACC|6i1QjO#59Tc3gA8PKj=8ubhfSxWj(ht6q}*c_uNB) zP<~1+wKtjUS3b6bh)h8?yzj-xE=pcRMeVWX-y^?GD;M_tG?2BNFc7Kl%LyZaH!;KJ zS3wX(fiVHgkV<<%1NGt4b1b=^N`alm=v^NGyx>4v-C2yw^gP-ecSE0ICN2%R_v!f~Gq>|nt zc^1w;%v4gmr(ifo#I!Q(k=239fF~OGfor5@DtbU+f@ObRZZij^5&7dexMAUC1yStsjx^kTSbjI*!fBAh+Kk z2+fW9QO2cw6WL6_&)s@)?crGZDyhPCUO|%ia_+OwX5*f0#j6shx6?nG9vBsL;wh$! zxv*XErhRQBsc_SwBL2d&z!(ZUXNH`JlD7)kQws;fBb3+@$uzBcb`{XVuYk{cWWCp`Dk$zgcWQW`&iRyG-T7c)tUj#T^O^9seBe!mM8N%iBL{*# zK1@j_=!J)8pC3X#Add5Z?pyE--8ve0*&zw+TLKOXh~14*eS*zkrGX$G6MRf| zpHUR+00J2>^ss#vR0THJw#*%(U6k3{Z5+C75TwFx^94ctqbwQlGRJOf#j)6nr_1O^ z+SWfVnN>aQBRu~%XAOifPP(&BUkC77T>a*-rys0Tim39IiAGN%}-^Co1Z95cnbOQYa2=CH<45g;hC9WzK>3zHxI7;AJqzQ8dg< rNq+b;{n&k8lJwt28H