diff --git a/.circleci/config.yml b/.circleci/config.yml index a8df5dab..4b050d42 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,91 +1,34 @@ -# See: https://circleci.com/docs/2.0/language-python/ +version: 2.1 -version: 2 -jobs: +jobs: build-docs: - working_directory: ~/repo docker: - - image: cimg/python:3.8 - + - image: cimg/python:3.13-node steps: - checkout - run: name: Install Python dependencies command: | - python3 -m venv venv - source venv/bin/activate - pip install --upgrade pip wheel setuptools - pip install -r site/requirements.txt -r requirements.txt - - - restore_cache: - keys: - - cache-data + python -m pip install --upgrade pip tox - run: - name: Build site - no_output_timeout: 30m + name: Build documentation + no_output_timeout: 5m + environment: + # Ensure this is same as store_artifacts path below + DOCS_PATH: _build/html command: | - # NOTE: blas multithreading behaves badly on circleci + export BASE_URL="/output/job/$CIRCLE_WORKFLOW_JOB_ID/artifacts/0/$DOCS_PATH" export OMP_NUM_THREADS=1 - source venv/bin/activate - # n = nitpicky (broken links), W = warnings as errors, - # T = full tracebacks, keep-going = run to completion even with errors - make -C site/ SPHINXOPTS="-nWT --keep-going" html - - - save_cache: - key: cache-data - paths: - - _data + python -m tox -e py313-buildhtml - store_artifacts: - path: site/_build/html - - - persist_to_workspace: - root: site/_build - paths: html - - deploy-docs: - working_directory: ~/repo - docker: - - image: circleci/python:3.8.5-buster - steps: - - checkout - - - attach_workspace: - at: site/_build - - - run: - name: install deploy deps - command : | - python3 -m pip install --user ghp-import - - - run: - name: configure git - command: | - git config --global user.name "ci-doc-deploy-bot" - git config --global user.email "ci-doc-deploy-bot@nomail" - git config --global push.default simple - - - add_ssh_keys: - fingerprints: - db:84:df:44:ad:77:d0:aa:2d:81:c9:73:30:9d:21:37 - - - run: - name: deploy to gh-pages - command: | - ghp-import -n -f -p -m "[skip ci] docs build of $CIRCLE_SHA1" site/_build/html - + path: _build/html workflows: version: 2 - build: + build-and-docs: jobs: - build-docs - - deploy-docs: - requires: - - build-docs - filters: - branches: - only: main diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..8db2976b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: ".github/workflows" + schedule: + interval: "monthly" + groups: + actions: + patterns: + - "*" + labels: + - "infrastructure" diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml new file mode 100644 index 00000000..e80599be --- /dev/null +++ b/.github/workflows/ci_publish.yml @@ -0,0 +1,46 @@ +name: Build and Publish HTML and deployed_notebooks + +on: + push: + branches: + - main + schedule: + - cron: '0 5 * * 1' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # `BASE_URL` determines, relative to the root of the domain, the URL that your site is served from. + # E.g., if your site lives at `https://mydomain.org/myproject`, set `BASE_URL=/myproject`. + # If, instead, your site lives at the root of the domain, at `https://mydomain.org`, set `BASE_URL=''`. + BASE_URL: /${{ github.event.repository.name }} + +jobs: + + publish_html: + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + name: Publish HTML + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 #v6.0.0 + + - name: Setup Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 #v6.1.0 + with: + python-version: '3.12' + + - name: Install dependencies + run: python -m pip install --upgrade tox + + - name: Execute notebooks while building HTMLs + run: tox -e py312-buildhtml + + - name: Publish + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./_build/html/ + commit_message: ${{ github.event.head_commit.message }} diff --git a/.github/workflows/ci_tests_run_notebooks.yml b/.github/workflows/ci_tests_run_notebooks.yml new file mode 100644 index 00000000..57126397 --- /dev/null +++ b/.github/workflows/ci_tests_run_notebooks.yml @@ -0,0 +1,59 @@ +name: Test notebooks + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '0 5 * * 1' + workflow_dispatch: + +jobs: + tests: + name: ${{ matrix.os }} ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Run all supported OS for one Python version, then add a few extra scenarios + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ['3.13'] + toxenv: [py313-test] + name: ['with Python 3.13',] + + include: + - python-version: '3.10' + toxenv: py310-test-oldestdeps + name: with Python 3.10 and oldest versioned dependencies + os: ubuntu-latest + + - python-version: '3.11' + toxenv: py311-test + name: with Python 3.11 and latest released version of dependencies + os: ubuntu-latest + + - python-version: '3.12' + toxenv: py312-test-predeps + name: with Python 3.12 and latest or pre-release version of dependencies + os: ubuntu-latest + + - python-version: '3.14-dev' + toxenv: py314-test-devdeps + name: with Python 3.14 and developer versioned dependencies + os: ubuntu-latest + + steps: + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 #v6.1.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: python -m pip install --upgrade tox + + - name: Test with nbval + run: tox ${{ matrix.toxargs }} -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }} diff --git a/.github/workflows/circleci-artifacts-redirector.yml b/.github/workflows/circleci-artifacts-redirector.yml new file mode 100644 index 00000000..f24efe72 --- /dev/null +++ b/.github/workflows/circleci-artifacts-redirector.yml @@ -0,0 +1,16 @@ +name: Run CircleCI artifacts redirector for rendered HTML +on: [status] +jobs: + circleci_artifacts_redirector_job: + runs-on: ubuntu-latest + name: Run CircleCI artifacts redirector + steps: + - name: GitHub Action step + uses: scientific-python/circleci-artifacts-redirector-action@5d358ff96e96429a5c64a969bb4a574555439f4f # v1.3.1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + api-token: ${{ secrets.CIRCLE_TOKEN }} + artifact-path: 0/_build/html/index.html + circleci-jobs: build-docs + domain: circle.scientific-python.dev + job-title: "--> Rendering Preview <--" diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml deleted file mode 100644 index 5115024b..00000000 --- a/.github/workflows/circleci.yml +++ /dev/null @@ -1,12 +0,0 @@ -on: [status] -jobs: - circleci_artifacts_redirector_job: - runs-on: ubuntu-latest - name: Run CircleCI artifacts redirector - steps: - - name: GitHub Action step - uses: larsoner/circleci-artifacts-redirector-action@master - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - artifact-path: 0/site/_build/html/index.html - circleci-jobs: build-docs diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml deleted file mode 100644 index c58fbc9a..00000000 --- a/.github/workflows/conda.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build site - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - test: - runs-on: ${{ matrix.os }}-latest - - strategy: - matrix: - os: [ubuntu, macos, windows] - - defaults: - run: - shell: bash -l {0} - - steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - activate-environment: numpy-tutorials - environment-file: environment.yml - miniforge-variant: Mambaforge - miniforge-version: latest - use-mamba: true - python-version: "3.10" - auto-activate-base: false - - name: inspect and build - id: build_step - continue-on-error: true - run: | - conda info - conda list - make -C site/ SPHINXOPTS="-nWT --keep-going" html - - - uses: actions/upload-artifact@v2 - with: - name: sphinx-build-artifact - path: site/_build/html/reports - - - name: fail on build errors - if: steps.build_step.outcome != 'success' - run: exit 1 - diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml deleted file mode 100644 index 9489bff9..00000000 --- a/.github/workflows/notebooks.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Test notebooks - -on: - push: - branches: - - main - pull_request: - branches: - - main - schedule: - - cron: '0 5 * * 1' - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - max-parallel: 12 - matrix: - os: [Ubuntu-20.04, macOS-latest] - python-version: [3.8, 3.9, "3.10"] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install -r site/requirements.txt -r requirements.txt - python -m pip list - - - name: Test with nbval - run: | - python -m pip install pytest nbval - find content/ -name "*.md" -exec jupytext --to notebook {} \; - # TODO: find better way to exclude notebooks from test - rm content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.ipynb - rm content/pairing.ipynb - rm content/tutorial-style-guide.ipynb - rm content/tutorial-nlp-from-scratch.ipynb - # Test notebook execution - pytest --nbval-lax --durations=10 content/ diff --git a/.gitignore b/.gitignore index fe8151b1..63354e46 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ pmip tags cscope.out .ipynb_checkpoints +.tox # Compiled source # ################### @@ -97,3 +98,5 @@ site/notebooks/* content/mooreslaw_regression* content/tutorial-x-ray-image-processing/xray_image.gif content/video +content/*ipynb +content/x_y-squared* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..13ed7d44 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,117 @@ +# Contributing + +We very much welcome contributions! If you have an idea or proposal for a new +tutorial, please [open an issue](https://github.com/numpy/numpy-tutorials/issues) +with an outline. + +Don’t worry if English is not your first language, or if you can only come up +with a rough draft. Open source is a community effort. Do your best – we’ll help +fix issues. + +Images and real-life data make text more engaging and powerful, but be sure what +you use is appropriately licensed and available. Here again, even a rough idea +for artwork can be polished by others. + +## Building the website + +```{note} +The NumPy tutorials are powered by [`jupyter-book`][jb-docs] and the +[`MyST` document engine][mystmd]. +See the linked documentation for further details. +``` + +[jb-docs]: https://jupyterbook.org/stable/ +[mystmd]: https://mystmd.org/ + +### Quickstart + +Set up a development environment with the dependencies listed in +`requirements.txt` and `site/requirements.txt`. +For example, using the built-in [`venv`][venv] module: + +```bash +python -m venv np-tutorials +source np-tutorials/bin/activate +python -m pip install -r requirements.txt -r site/requirements.txt +``` + +[venv]: https://docs.python.org/3/library/venv.html + +The site can then be built with: + +```bash +jupyter-book start --execute +``` + +This will execute all the notebooks and start a web server to view the rendered +content locally. +View the rendered site by opening the ``localhost:30xy`` in your preferred browser (the exact port number will be printed in your terminal). + +## Adding your own tutorials + +If you have your own tutorial in the form of a Jupyter notebook (an `.ipynb` +file) and you'd like to try add it out to the repository, follow the steps below. + +### Create an issue + +Go to and create a new issue +with your proposal. +Give as much detail as you can about what kind of content you would like to +write (tutorial, how-to) and what you plan to cover. +We will try to respond as quickly as possible with comments, if applicable. + +### Check out our suggested template + +You can use this template to make your content consistent with our existing +tutorials. + +### Upload your content + +Remember to clear all outputs on your notebook before uploading it. + + + +For more information about GitHub and its workflow, you can see +[this document][collab]. + +[collab]: https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests diff --git a/LICENSE.txt b/LICENSE.txt index 8ce64521..014d51c9 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2005-2020, NumPy Developers. +Copyright (c) 2005-2023, NumPy Developers. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index b5453a6e..5109e2c9 100644 --- a/README.md +++ b/README.md @@ -20,141 +20,10 @@ or navigate to any of the documents listed below and download it individually. 3. [Tutorial: Saving and sharing your NumPy arrays](content/save-load-arrays.md) 4. [Tutorial: NumPy deep learning on MNIST from scratch](content/tutorial-deep-learning-on-mnist.md) 5. [Tutorial: X-ray image processing](content/tutorial-x-ray-image-processing.md) -6. [Tutorial: NumPy deep reinforcement learning with Pong from pixels](content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md) -7. [Tutorial: Masked Arrays](content/tutorial-ma.md) -8. [Tutorial: Static Equilibrium](content/tutorial-static_equilibrium.md) -9. [Tutorial: Plotting Fractals](content/tutorial-plotting-fractals.ipynb) -10. [Tutorial: NumPy natural language processing from scratch with a focus on ethics](content/tutorial-nlp-from-scratch.md) -11. [Tutorial: Analysing the impact of the lockdown on air quality in Delhi, India](content/tutorial-air-quality-analysis.md) - - -## Contributing - -We very much welcome contributions! If you have an idea or proposal for a new -tutorial, please [open an issue](https://github.com/numpy/numpy-tutorials/issues) -with an outline. - -Don’t worry if English is not your first language, or if you can only come up -with a rough draft. Open source is a community effort. Do your best – we’ll help -fix issues. - -Images and real-life data make text more engaging and powerful, but be sure what -you use is appropriately licensed and available. Here again, even a rough idea -for artwork can be polished by others. - -The NumPy tutorials are a curated collection of -[MyST-NB](https://myst-nb.readthedocs.io/) notebooks. These notebooks are used -to produce static websites and can be opened as notebooks in Jupyter using -[Jupytext](https://jupytext.readthedocs.io). - -> __Note:__ You should use [CommonMark](https://commonmark.org) markdown -> cells. Jupyter only renders CommonMark. - -### Why Jupyter Notebooks? - -The choice of Jupyter Notebook in this repo instead of the usual format -([reStructuredText, through Sphinx](https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html)) -used in the main NumPy documentation has two reasons: - - * Jupyter notebooks are a common format for communicating scientific - information. - * Jupyter notebooks can be launched in [Binder](https://www.mybinder.org), so that users can interact - with tutorials - * rST may present a barrier for some people who might otherwise be very - interested in contributing tutorial material. - -#### Note - -You may notice our content is in markdown format (`.md` files). We review and -host notebooks in the [MyST-NB](https://myst-nb.readthedocs.io/) format. We -accept both Jupyter notebooks (`.ipynb`) and MyST-NB notebooks (`.md`). If you want -to sync your `.ipynb` to your `.md` file follow the [pairing -tutorial](content/pairing.md). - -### Adding your own tutorials - -If you have your own tutorial in the form of a Jupyter notebook (a `.ipynb` -file) and you'd like to add it to the repository, follow the steps below. - - -#### Create an issue - -Go to [https://github.com/numpy/numpy-tutorials/issues](https://github.com/numpy/numpy-tutorials/issues) -and create a new issue with your proposal. Give as much detail as you can about -what kind of content you would like to write (tutorial, how-to) and what you -plan to cover. We will try to respond as quickly as possible with comments, if -applicable. - -#### Check out our suggested template - -You can use our [Tutorial Style Guide](content/tutorial-style-guide.md) to make -your content consistent with our existing tutorials. - -#### Upload your content - - - -For more information about GitHub and its workflow, you can see -[this document](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests). - - -### Building the Sphinx site locally - -Building the tutorials website, which is published at -https://github.com/numpy/numpy-tutorials, locally isn't necessary before making -a contribution, but may be helpful: - -```bash -conda env create -f environment.yml -conda activate numpy-tutorials -cd site -make html -``` +6. [Tutorial: Masked Arrays](content/tutorial-ma.md) +7. [Tutorial: Static Equilibrium](content/tutorial-static_equilibrium.md) +8. [Tutorial: Plotting Fractals](content/tutorial-plotting-fractals.ipynb) +9. [Tutorial: Analysing the impact of the lockdown on air quality in Delhi, India](content/tutorial-air-quality-analysis.md) ## Translations diff --git a/content/_static/01-classic.gif b/content/_static/01-classic.gif deleted file mode 100644 index 7fc540ba..00000000 Binary files a/content/_static/01-classic.gif and /dev/null differ diff --git a/content/_static/02-jupyterlab.gif b/content/_static/02-jupyterlab.gif deleted file mode 100644 index f4f040bb..00000000 Binary files a/content/_static/02-jupyterlab.gif and /dev/null differ diff --git a/content/_static/lstm.gif b/content/_static/lstm.gif deleted file mode 100644 index a2884ee3..00000000 Binary files a/content/_static/lstm.gif and /dev/null differ diff --git a/content/_static/mem_block.png b/content/_static/mem_block.png deleted file mode 100644 index 58df4536..00000000 Binary files a/content/_static/mem_block.png and /dev/null differ diff --git a/content/_static/tutorial-deep-reinforcement-learning-with-pong-from-pixels.png b/content/_static/tutorial-deep-reinforcement-learning-with-pong-from-pixels.png deleted file mode 100644 index e76908a9..00000000 Binary files a/content/_static/tutorial-deep-reinforcement-learning-with-pong-from-pixels.png and /dev/null differ diff --git a/content/mooreslaw-tutorial.md b/content/mooreslaw-tutorial.md index 18a13dcc..638853c4 100644 --- a/content/mooreslaw-tutorial.md +++ b/content/mooreslaw-tutorial.md @@ -1,4 +1,5 @@ --- +short_title: Moore's Law jupytext: text_representation: extension: .md @@ -21,8 +22,7 @@ _The number of transistors reported per a given chip plotted on a log scale in t In 1965, engineer Gordon Moore [predicted](https://en.wikipedia.org/wiki/Moore%27s_law) that transistors on a chip would double every two years in the coming decade -[[1](https://en.wikipedia.org/wiki/Moore%27s_law), -[2](https://newsroom.intel.com/wp-content/uploads/sites/11/2018/05/moores-law-electronics.pdf)]. +[[1](https://en.wikipedia.org/wiki/Moore%27s_law)]. You'll compare Moore's prediction against actual transistor counts in the 53 years following his prediction. You will determine the best-fit constants to describe the exponential growth of transistors on semiconductors compared to Moore's Law. @@ -44,27 +44,24 @@ the 53 years following his prediction. You will determine the best-fit constants * NumPy * [Matplotlib](https://matplotlib.org/) -* [statsmodels](https://www.statsmodels.org) ordinary linear regression imported with the following commands ```{code-cell} import matplotlib.pyplot as plt import numpy as np -import statsmodels.api as sm ``` **2.** Since this is an exponential growth law you need a little background in doing math with [natural logs](https://en.wikipedia.org/wiki/Natural_logarithm) and [exponentials](https://en.wikipedia.org/wiki/Exponential_function). -You'll use these NumPy, Matplotlib, and statsmodels functions: +You'll use these NumPy and Matplotlib functions: * [`np.loadtxt`](https://numpy.org/doc/stable/reference/generated/numpy.loadtxt.html): this function loads text into a NumPy array * [`np.log`](https://numpy.org/doc/stable/reference/generated/numpy.log.html): this function takes the natural log of all elements in a NumPy array * [`np.exp`](https://numpy.org/doc/stable/reference/generated/numpy.exp.html): this function takes the exponential of all elements in a NumPy array * [`lambda`](https://docs.python.org/3/library/ast.html?highlight=lambda#ast.Lambda): this is a minimal function definition for creating a function model -* [`plt.semilogy`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.semilogy.html): this function will plot x-y data onto a figure with a linear x-axis and $\log_{10}$ y-axis -[`plt.plot`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.plot.html): this function will plot x-y data on linear axes -* [`sm.OLS`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html): find fitting parameters and standard errors using the statsmodels ordinary least squares model +* [`plt.semilogy`](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.semilogy.html): this function will plot x-y data onto a figure with a linear x-axis and $\log_{10}$ y-axis +[`plt.plot`](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.plot.html): this function will plot x-y data on linear axes * slicing arrays: view parts of the data loaded into the workspace, slice the arrays e.g. `x[:10]` for the first 10 values in the array, `x` * boolean array indexing: to view parts of the data that match a given condition use boolean operations to index an array * [`np.block`](https://numpy.org/doc/stable/reference/generated/numpy.block.html): to combine arrays into 2D arrays @@ -110,7 +107,7 @@ $B_M=-675.4$ Since the function represents Moore's law, define it as a Python function using -[`lambda`](https://docs.python.org/3/library/ast.html?highlight=lambda#ast.Lambda) +[`lambda`](https://docs.python.org/3/library/ast.html?highlight=lambda#ast.Lambda): ```{code-cell} A_M = np.log(2) / 2 @@ -133,7 +130,7 @@ print("This is x{:.2f} more transistors than 1971".format(ML_1973 / ML_1971)) Now, make a prediction based upon the historical data for semiconductors per chip. The [Transistor Count -\[4\]](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) +\[3\]](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) each year is in the `transistor_data.csv` file. Before loading a \*.csv file into a NumPy array, its a good idea to inspect the structure of the file first. Then, locate the columns of interest and save them to a @@ -160,7 +157,7 @@ The extra options below will put the data in the desired format: * `delimiter = ','`: specify delimeter as a comma ',' (this is the default behavior) * `usecols = [1,2]`: import the second and third columns from the csv -* `skiprows = 1`: do not use the first row, because its a header row +* `skiprows = 1`: do not use the first row, because it's a header row ```{code-cell} data = np.loadtxt("transistor_data.csv", delimiter=",", usecols=[1, 2], skiprows=1) @@ -186,7 +183,7 @@ print("trans. cnt:\t", transistor_count[:10]) You are creating a function that predicts the transistor count given a year. You have an _independent variable_, `year`, and a _dependent -variable_, `transistor_count`. Transform the independent variable to +variable_, `transistor_count`. Transform the dependent variable to log-scale, $y_i = \log($ `transistor_count[i]` $),$ @@ -215,59 +212,31 @@ where $\mathbf{y}$ are the observations of the log of the number of transistors in a 1D array and $\mathbf{Z}=[\text{year}_i^1,~\text{year}_i^0]$ are the polynomial terms for $\text{year}_i$ in the first and second columns. By creating this set of regressors in the $\mathbf{Z}-$matrix you set -up an ordinary least squares statistical model. Some clever -NumPy array features will build $\mathbf{Z}$ +up an ordinary least squares statistical model. -1. `year[:,np.newaxis]` : takes the 1D array with shape `(179,)` and turns it into a 2D column vector with shape `(179,1)` -2. `**[1, 0]` : stacks two columns, in the first column is `year**1` and the second column is `year**0 == 1` +`Z` is a linear model with two parameters, i.e. a polynomial with degree `1`. +Therefore we can represent the model with `numpy.polynomial.Polynomial` and +use the fitting functionality to determine the model parameters: ```{code-cell} -Z = year[:, np.newaxis] ** [1, 0] +model = np.polynomial.Polynomial.fit(year, yi, deg=1) ``` -Now that you have the created a matrix of regressors, $\mathbf{Z},$ and -the observations are in vector, $\mathbf{y},$ you can use these -variables to build the an ordinary least squares model with -[`sm.OLS`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html). +By default, `Polynomial.fit` performs the fit in the domain determined by the +independent variable (`year` in this case). +The coefficients for the unscaled and unshifted model can be recovered with the +`convert` method: -```{code-cell} -model = sm.OLS(yi, Z) -``` - -Now, you can view the fitting constants, $A$ and $B$, and their standard -errors. Run the -[`fit`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.OLS.html) and print the -[`summary`](https://www.statsmodels.org/stable/generated/statsmodels.regression.linear_model.RegressionResults.summary.html) to view results as such, ```{code-cell} -results = model.fit() -print(results.summary()) +model = model.convert() +model ``` -The __OLS Regression Results__ summary gives a lot of information about -the regressors, $\mathbf{Z},$ and observations, $\mathbf{y}.$ The most -important outputs for your current analysis are - -``` -================================= - coef std err ---------------------------------- -x1 0.3416 0.006 -const -666.3264 11.890 -================================= -``` -where `x1` is slope, $A=0.3416$, `const` is the intercept, -$B=-666.364$, and `std error` gives the precision of constants -$A=0.342\pm 0.006~\dfrac{\log(\text{transistors}/\text{chip})}{\text{years}}$ and $B=-666\pm -12~\log(\text{transistors}/\text{chip}),$ where the units are in -$\log(\text{transistors}/\text{chip})$. You created an exponential growth model. -To get the constants, save them to an array `AB` with -`results.params` and assign $A$ and $B$ to `x1` and `constant`. +The individual parameters $A$ and $B$ are the coefficients of our linear model: ```{code-cell} -AB = results.params -A = AB[0] -B = AB[1] +B, A = model ``` Did manufacturers double the transistor count every two years? You have @@ -277,30 +246,20 @@ $\dfrac{\text{transistor_count}(\text{year} +2)}{\text{transistor_count}(\text{y \dfrac{e^{B}e^{A( \text{year} + 2)}}{e^{B}e^{A \text{year}}} = e^{2A}$ where increase in number of transistors is $xFactor,$ number of years is -2, and $A$ is the best fit slope on the semilog function. The error in -your -prediction, $\Delta(xFactor),$ comes from the precision of your constant -$A,$ which you calculated as the standard error $\Delta A= 0.006$. - -$\Delta (xFactor) = \frac{\partial}{\partial A}(e^{2A})\Delta A = 2Ae^{2A}\Delta A$ +2, and $A$ is the best fit slope on the semilog function. ```{code-cell} -print("Rate of semiconductors added on a chip every 2 years:") -print( - "\tx{:.2f} +/- {:.2f} semiconductors per chip".format( - np.exp((A) * 2), 2 * A * np.exp(2 * A) * 0.006 - ) -) +print(f"Rate of semiconductors added on a chip every 2 years: {np.exp(2 * A):.2f}") ``` Based upon your least-squares regression model, the number of -semiconductors per chip increased by a factor of $1.98\pm 0.01$ every two +semiconductors per chip increased by a factor of $1.98$ every two years. You have a model that predicts the number of semiconductors each year. Now compare your model to the actual manufacturing reports. Plot the linear regression results and all of the transistor counts. Here, use -[`plt.semilogy`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.semilogy.html) +[`plt.semilogy`](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.semilogy.html) to plot the number of transistors on a log-scale and the year on a linear scale. You have defined a three arrays to get to a final model @@ -321,11 +280,11 @@ $\text{transistor_count}_{\text{predicted}} = e^Be^{A\cdot \text{year}}$. +++ In the next plot, use the -[`fivethirtyeight`](https://matplotlib.org/3.1.1/gallery/style_sheets/fivethirtyeight.html) +[`fivethirtyeight`](https://matplotlib.org/gallery/style_sheets/fivethirtyeight.html) style sheet. The style sheet replicates -https://fivethirtyeight.com elements. Change the matplotlib style with -[`plt.style.use`](https://matplotlib.org/3.3.2/api/style_api.html#matplotlib.style.use). + elements. Change the matplotlib style with +[`plt.style.use`](https://matplotlib.org/api/style_api.html#matplotlib.style.use). ```{code-cell} transistor_count_predicted = np.exp(B) * np.exp(A * year) @@ -370,13 +329,13 @@ $\text{transistor_count} = e^{B}e^{A\cdot \text{year}}$. A great way to compare these measurements is to compare your prediction and Moore's prediction to the average transistor count and look at the range of reported values for that year. Use the -[`plt.plot`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.plot.html) +[`plt.plot`](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.plot.html) option, -[`alpha=0.2`](https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.artist.Artist.set_alpha.html), +[`alpha=0.2`](https://matplotlib.org/api/_as_gen/matplotlib.artist.Artist.set_alpha.html), to increase the transparency of the data. The more opaque the points appear, the more reported values lie on that measurement. The green $+$ is the average reported transistor count for 2017. Plot your predictions -for $\pm\frac{1}{2}~years. +for $\pm\frac{1}{2}$ years. ```{code-cell} transistor_count2017 = transistor_count[year == 2017] @@ -428,7 +387,7 @@ array using `np.loadtxt`, to save your model use two approaches ### Zipping the arrays into a file Using `np.savez`, you can save thousands of arrays and give them names. The function `np.load` will load the arrays back into the workspace as a -dictionary. You'll save a five arrays so the next user will have the year, +dictionary. You'll save five arrays so the next user will have the year, transistor count, predicted transistor count, Gordon Moore's predicted count, and fitting constants. Add one more variable that other users can use to understand the model, `notes`. @@ -455,7 +414,7 @@ np.savez( transistor_count=transistor_count, transistor_count_predicted=transistor_count_predicted, transistor_Moores_law=transistor_Moores_law, - regression_csts=AB, + regression_csts=(A, B), ) ``` @@ -561,7 +520,7 @@ double every two years from 1965 through 1975, but the average growth has maintained a consistent increase of $\times 1.98 \pm 0.01$ every two years from 1971 through 2019. In 2015, Moore revised his prediction to say Moore's law should hold until 2025. -[[3](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress)]. +[[2](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress)]. You can share these results as a zipped NumPy array file, `mooreslaw_regression.npz`, or as another csv, `mooreslaw_regression.csv`. The amazing progress in semiconductor @@ -574,6 +533,5 @@ has been over the last half-century. ## References 1. ["Moore's Law." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Moore%27s_law) -2. [Moore, Gordon E. (1965-04-19). "Cramming more components onto integrated circuits". intel.com. Electronics Magazine. Retrieved April 1, 2020.](https://newsroom.intel.com/wp-content/uploads/sites/11/2018/05/moores-law-electronics.pdf) -3. [Courtland, Rachel. "Gordon Moore: The Man Whose Name Means Progress." IEEE Spectrum. 30 Mar. 2015.](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress). -4. ["Transistor Count." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) +2. [Courtland, Rachel. "Gordon Moore: The Man Whose Name Means Progress." IEEE Spectrum. 30 Mar. 2015.](https://spectrum.ieee.org/computing/hardware/gordon-moore-the-man-whose-name-means-progress). +3. ["Transistor Count." Wikipedia article. Accessed Oct. 1, 2020.](https://en.wikipedia.org/wiki/Transistor_count#Microprocessors) diff --git a/content/pairing.md b/content/pairing.md deleted file mode 100644 index 573f15d4..00000000 --- a/content/pairing.md +++ /dev/null @@ -1,247 +0,0 @@ ---- -jupytext: - formats: ipynb,md:myst - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.11.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -# Pairing Jupyter notebooks and MyST-NB - -## What you'll do -This guide will keep a Jupyter notebook synced _or paired_ between -`.ipynb` and `.md`. - -## What you'll learn -- The difference between Jupyter's json format and MyST-NB's markdown - format -- The benefits and drawbacks of json and markdown -- How to keep `.ipynb` and `.md` files in sync - -## What you'll need -- [Jupyter](https://jupyter.org/) -- [Jupytext](https://jupytext.readthedocs.io/en/latest/index.html) - ---- -## Background - -The [NumPy tutorials](https://github.com/numpy/numpy-tutorials) are -reviewed and executed as [MyST-NB](https://myst-nb.readthedocs.io/) -notebooks. Content is easier to review in this markdown format. You can -keep your `.ipynb` in sync with the content on NumPy tutorials. The -NumPy tutorials use -[Jupytext](https://jupytext.readthedocs.io/en/latest/index.html) to -convert your `.ipynb` file to [MyST -Markdown](https://github.com/mwouts/jupytext/blob/master/docs/formats.md#myst-markdown) -format. - -Jupyter notebooks are stored on your disk in a -[json format](https://nbformat.readthedocs.io/en/latest/format_description.html). The json format is -very powerful and allows you to store almost any input and output that -Python libraries can create. The drawback is that it is hard to see and compare changes made in the notebook file when reviewing pull requests, because this means the reviewers are looking only at the raw json files. - -MyST-NB notebooks are stored on your disk in a -[markdown](https://en.wikipedia.org/wiki/Markdown) format. The markdown -format is a lightweight markup language. Its key design goal is -[_readability_](https://daringfireball.net/projects/markdown/syntax#philosophy). -The drawback is that markdown can only store the inputs of your code. -Each time you open the notebook, you must execute the inputs to see the -output. - -> __Note:__ You should use [common mark](https://commonmark.org) -> markdown cells. Jupyter only renders common mark markdown, but MyST-NB -> supports a variety of restructured text directives. These Sphinx -> markdown directives will render when NumPy tutorials are built into a -> static website, but they will show up as raw code when you open in -> Jupyter locally or on [Binder](https://mybinder.org). - -Consider these two versions of the same __Simple notebook example__. You -have three things in the notebooks: - -1. A markdown cell that explains the code - ```This code calculates 2+2 and prints the output.``` -2. A code cell that shows the code - ```python - x = 2 + 2 - print('x = ', x) - ``` -3. The output of the code cell - ```python - x = 4 - ``` ---- -__
Simple notebook example
__ -This code calculates 2+2 and prints the output. - -```{code-cell} -x = 2 + 2 -print("x = ", x) -``` - ---- - -Here are the two Simple notebook example raw inputs side-by-side: - - - - - - - - - - - -
json .ipynbMyST-NB .md
- -```json -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This code calculates 2+2 and prints the output" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "x = 4\n" - ] - } - ], - "source": [ - "x = 2 + 2\n", - "print('x = ', x)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} -``` - - - -```` ---- -jupytext: - formats: ipynb,md:myst - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.6.0 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -This code calculates 2+2 and prints the output - -```{code-cell} ipython3 -x = 2 + 2 -print('x = ', x) -``` -```` -
- -The MyST-NB `.md` is much shorter, but it does not save the output `4`. - - -## Pair your notebook files `.ipynb` and `.md` - -When you submit a Jupyter notebook to NumPy tutorials, we (the reviewers) will convert -it to a MyST-NB format. You can also submit the MyST-NB `.md` in your -pull request. -To keep the `.ipynb` and `.md` in sync--_or paired_--you need -[Jupytext](https://jupytext.readthedocs.io/en/latest/index.html). - -Install `jupytext` using: - -``` -pip install jupytext -``` -or -``` -conda install jupytext -c conda-forge -``` - -Once installed, start your `jupyter lab` or `jupyter notebook` -session in the browser. When launching `jupyter lab` it will ask you to rebuild -to include the Jupytext extension. - -You can pair the two formats in the classic Jupyter, Jupyter Lab, -or the command line: - -```{admonition} **1. Classic Jupyter Jupytext pairing** -:class: toggle - -![Animation showing pairing with Jupyter classic](_static/01-classic.gif) -``` - -```{admonition} **2. JupyterLab Jupytext pairing** -:class: toggle - -![Animation showing pairing with JupyterLab](_static/02-jupyterlab.gif) -``` - -````{admonition} **3. Command line Jupytext pairing** -:class: toggle - -```sh -jupytext --set-formats ipynb,myst notebook.ipynb -``` - -Then, update either the MyST markdown or notebook file: - -```sh -jupytext --sync notebook.ipynb -``` -```` - -> __Note:__ With Jupytext installed, the classic Jupyter interface will -> automatically open MyST files as notebooks. In JupyterLab, you can -> right-click and choose "Open With -> Notebook" to open as a notebook. -> The outputs of your code cells are only saved in the `.ipynb` file. - -## Wrapping up - -In this tutorial, you saw the json `.ipynb` and MyST-NB `.md` raw code -to create Jupyter notebooks. You can use both formats to create -tutorials. Now you can work in either a simple text editor like VIM -or emacs or continue building notebooks in your browser. Jupytext can -handle pairing to keep your work in sync. diff --git a/content/save-load-arrays.md b/content/save-load-arrays.md index 1960d0de..704f3475 100644 --- a/content/save-load-arrays.md +++ b/content/save-load-arrays.md @@ -1,4 +1,5 @@ --- +short_title: Sharing Array Data jupytext: text_representation: extension: .md @@ -68,7 +69,7 @@ will assign `x` to the integers from 0 to 9 using [`np.arange`](https://numpy.org/doc/stable/reference/generated/numpy.arange.html). ```{code-cell} -x = np.arange(0, 10, 1) +x = np.arange(10) y = x ** 2 print(x) print(y) @@ -127,7 +128,7 @@ print(load_xy.files) ``` ```{code-cell} -whos +%whos ``` ## Reassign the NpzFile arrays to `x` and `y` @@ -187,18 +188,8 @@ np.savetxt("x_y-squared.csv", X=array_out, header="x, y", delimiter=",") Open the file, `x_y-squared.csv`, and you'll see the following: -``` -# x, y -0.000000000000000000e+00,0.000000000000000000e+00 -1.000000000000000000e+00,1.000000000000000000e+00 -2.000000000000000000e+00,4.000000000000000000e+00 -3.000000000000000000e+00,9.000000000000000000e+00 -4.000000000000000000e+00,1.600000000000000000e+01 -5.000000000000000000e+00,2.500000000000000000e+01 -6.000000000000000000e+00,3.600000000000000000e+01 -7.000000000000000000e+00,4.900000000000000000e+01 -8.000000000000000000e+00,6.400000000000000000e+01 -9.000000000000000000e+00,8.100000000000000000e+01 +```{code-cell} +!head x_y-squared.csv ``` ## Our arrays as a csv file diff --git a/content/text_preprocessing.py b/content/text_preprocessing.py deleted file mode 100644 index cdfa3ad0..00000000 --- a/content/text_preprocessing.py +++ /dev/null @@ -1,188 +0,0 @@ -import pandas as pd -import argparse -import numpy as np -import re # (https://docs.python.org/3/library/re.html) for tokenising textual data -import string # (https://docs.python.org/3/library/string.html) for string operations - -# Creating the random instance -rng = np.random.default_rng() - -class TextPreprocess: - """Text Preprocessing for a Natural Language Processing model.""" - - - def cleantext(self, df, text_column, remove_stopwords = True, remove_punc = True): - """Function to clean text data by removing stopwords, tags and punctuation. - - Parameters - ---------- - df : pandas dataframe - The dataframe housing the input data. - text_column : str - Column in dataframe whose text is to be cleaned. - remove_stopwords : bool - if True, remove stopwords from text - remove_punc : bool - if True, remove punctuation suymbols from text - - Returns - ------- - Numpy array - Cleaned text. - - """ - data = df - # converting all characters to lowercase - data[text_column] = data[text_column].str.lower() - - # List of common stopwords taken from https://gist.github.com/sebleier/554280 - stopwords = [ "a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "as", "at", "be", "because", - "been", "before", "being", "below", "between", "both", "but", "by", "could", "did", "do", "does", "doing", "down", "during", - "each", "few", "for", "from", "further", "had", "has", "have", "having", "he", "he'd", "he'll", "he's", "her", "here", - "here's", "hers", "herself", "him", "himself", "his", "how", "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in", "into", - "is", "it", "it's", "its", "itself", "let's", "me", "more", "most", "my", "myself", "nor", "of", "on", "once", "only", "or", - "other", "ought", "our", "ours", "ourselves", "out", "over", "own", "same", "she", "she'd", "she'll", "she's", "should", - "so", "some", "such", "than", "that", "that's", "the", "their", "theirs", "them", "themselves", "then", "there", "there's", - "these", "they", "they'd", "they'll", "they're", "they've", "this", "those", "through", "to", "too", "under", "until", "up", - "very", "was", "we", "we'd", "we'll", "we're", "we've", "were", "what", "what's", "when", "when's", "where", "where's", - "which", "while", "who", "who's", "whom", "why", "why's", "with", "would", "you", "you'd", "you'll", "you're", "you've", - "your", "yours", "yourself", "yourselves" ] - - def remove_stopwords(data, column): - data[f'{column} without stopwords'] = data[column].apply(lambda x : ' '.join([word for word in x.split() if word not in (stopwords)])) - return data - - def remove_tags(string): - result = re.sub('<*>','',string) - return result - - # remove html tags and brackets from text - if remove_stopwords: - data_without_stopwords = remove_stopwords(data, text_column) - data_without_stopwords[f'clean_{text_column}']= data_without_stopwords[f'{text_column} without stopwords'].apply(lambda cw : remove_tags(cw)) - if remove_punc: - data_without_stopwords[f'clean_{text_column}'] = data_without_stopwords[f'clean_{text_column}'].str.replace('[{}]'.format(string.punctuation), ' ', regex = True) - - X = data_without_stopwords[f'clean_{text_column}'].to_numpy() - - return X - - def split_data (self, X, y, split_percentile): - """Function to split data into training and testing data. - - Parameters - ---------- - X : Numpy Array - Contains textual data. - y : Numpy Array - Contains target data. - split_percentile : int - Proportion of training to testing data. - - - Returns - ------- - Tuple - Contains numpy arrays of test and training data. - - """ - y = np.array(list(map(lambda x: 1 if x=="positive" else 0, y))) - arr_rand = rng.random(X.shape[0]) - split = arr_rand < np.percentile(arr_rand, split_percentile) - X_train = X[split] - y_train = y[split] - X_test = X[~split] - y_test = y[~split] - - return (X_train, y_train, X_test, y_test) - - - def sent_tokeniser (self, x): - """Function to split text into sentences. - - Parameters - ---------- - x : str - piece of text - - Returns - ------- - list - sentences with punctuation removed. - - """ - sentences = re.split(r'(? **Note:** In more technical terms, you: > @@ -544,17 +544,13 @@ for j in range(epochs): # Summarize error and accuracy metrics at each epoch print( - "\n" - + "Epoch: " - + str(j) - + " Training set error:" - + str(training_loss / float(len(training_images)))[0:5] - + " Training set accuracy:" - + str(training_accurate_predictions / float(len(training_images))) - + " Test set error:" - + str(test_loss / float(len(test_images)))[0:5] - + " Test set accuracy:" - + str(test_accurate_predictions / float(len(test_images))) + ( + f"Epoch: {j}\n" + f" Training set error: {training_loss / len(training_images):.3f}\n" + f" Training set accuracy: {training_accurate_predictions / len(training_images)}\n" + f" Test set error: {test_loss / len(test_images):.3f}\n" + f" Test set accuracy: {test_accurate_predictions / len(test_images)}" + ) ) ``` @@ -565,39 +561,31 @@ The training process may take many minutes, depending on a number of factors, su After executing the cell above, you can visualize the training and test set errors and accuracy for an instance of this training process. ```{code-cell} +epoch_range = np.arange(epochs) + 1 # Starting from 1 + # The training set metrics. -y_training_error = [ - store_training_loss[i] / float(len(training_images)) - for i in range(len(store_training_loss)) -] -x_training_error = range(1, len(store_training_loss) + 1) -y_training_accuracy = [ - store_training_accurate_pred[i] / float(len(training_images)) - for i in range(len(store_training_accurate_pred)) -] -x_training_accuracy = range(1, len(store_training_accurate_pred) + 1) +training_metrics = { + "accuracy": np.asarray(store_training_accurate_pred) / len(training_images), + "error": np.asarray(store_training_loss) / len(training_images), +} # The test set metrics. -y_test_error = [ - store_test_loss[i] / float(len(test_images)) for i in range(len(store_test_loss)) -] -x_test_error = range(1, len(store_test_loss) + 1) -y_test_accuracy = [ - store_training_accurate_pred[i] / float(len(training_images)) - for i in range(len(store_training_accurate_pred)) -] -x_test_accuracy = range(1, len(store_test_accurate_pred) + 1) +test_metrics = { + "accuracy": np.asarray(store_test_accurate_pred) / len(test_images), + "error": np.asarray(store_test_loss) / len(test_images), +} # Display the plots. fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(15, 5)) -axes[0].set_title("Training set error, accuracy") -axes[0].plot(x_training_accuracy, y_training_accuracy, label="Training set accuracy") -axes[0].plot(x_training_error, y_training_error, label="Training set error") -axes[0].set_xlabel("Epochs") -axes[1].set_title("Test set error, accuracy") -axes[1].plot(x_test_accuracy, y_test_accuracy, label="Test set accuracy") -axes[1].plot(x_test_error, y_test_error, label="Test set error") -axes[1].set_xlabel("Epochs") +for ax, metrics, title in zip( + axes, (training_metrics, test_metrics), ("Training set", "Test set") +): + # Plot the metrics + for metric, values in metrics.items(): + ax.plot(epoch_range, values, label=metric.capitalize()) + ax.set_title(title) + ax.set_xlabel("Epochs") + ax.legend() plt.show() ``` diff --git a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md b/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md deleted file mode 100644 index 3ef3de03..00000000 --- a/content/tutorial-deep-reinforcement-learning-with-pong-from-pixels.md +++ /dev/null @@ -1,664 +0,0 @@ ---- -jupytext: - formats: ipynb,md:myst - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.11.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -# Deep reinforcement learning with Pong from pixels - -This tutorial demonstrates how to implement a deep reinforcement learning (RL) agent from scratch using a policy gradient method that learns to play the [Pong](https://gym.openai.com/envs/Pong-v0/) video game using screen pixels as inputs with NumPy. Your Pong agent will obtain experience on the go using an [artificial neural network](https://en.wikipedia.org/wiki/Artificial_neural_network) as its [policy](https://en.wikipedia.org/wiki/Reinforcement_learning). - -Pong is a 2D game from 1972 where two players use "rackets" to play a form of table tennis. Each player moves the racket up and down the screen and tries to hit a ball in their opponent's direction by touching it. The goal is to hit the ball such that it goes past the opponent's racket (they miss their shot). According to the rules, if a player reaches 21 points, they win. In Pong, the RL agent that learns to play against an opponent is displayed on the right. - -![Diagram showing operations detailed in this tutorial](_static/tutorial-deep-reinforcement-learning-with-pong-from-pixels.png) - -This example is based on the [code](https://gist.github.com/karpathy/a4166c7fe253700972fcbc77e4ea32c5) developed by [Andrej Karpathy](https://karpathy.ai) for the [Deep RL Bootcamp](https://sites.google.com/view/deep-rl-bootcamp/home) in 2017 at UC Berkeley. His [blog post](http://karpathy.github.io/2016/05/31/rl/) from 2016 also provides more background on the mechanics and theory used in Pong RL. - -## Prerequisites - -- **OpenAI Gym**: To help with the game environment, you will use [Gym](https://gym.openai.com) — an open-source Python interface [developed by OpenAI](https://arxiv.org/abs/1606.01540) that helps perform RL tasks while supporting many simulation environments. -- **Python and NumPy**: The reader should have some knowledge of Python, NumPy array manipulation, and linear algebra. -- **Deep learning and deep RL**: You should be familiar with main concepts of [deep learning](https://en.wikipedia.org/wiki/Deep_learning), which are explained in the [Deep learning](http://www.cs.toronto.edu/~hinton/absps/NatureDeepReview.pdf) paper published in 2015 by Yann LeCun, Yoshua Bengio, and Geoffrey Hinton, who are regarded as some of the pioneers of the field. The tutorial will try to guide you through the main concepts of deep RL and you will find various literature with links to original sources for your convenience. -- **Jupyter notebook environments**: Because RL experiments can require high computing power, you can run the tutorial on the cloud for free using [Binder](https://mybinder.org) or [Google Colaboratory](https://colab.research.google.com/notebooks/intro.ipynb) (which offers free limited GPU and TPU acceleration). -- **Matplotlib**: For plotting images. Check out the [installation](https://matplotlib.org/3.3.3/users/installing.html) guide to set it up in your environment. - -This tutorial can also be run locally in an isolated environment, such as [Virtualenv](https://virtualenv.pypa.io/en/stable/) and [conda](https://docs.conda.io/). - -## Table of contents - -- A note on RL and deep RL -- Deep RL glossary -1. Set up Pong -2. Preprocess frames (the observation) -3. Create the policy (the neural network) and the forward pass -4. Set up the update step (backpropagation) -5. Define the discounted rewards (expected return) function -6. Train the agent for 3 episodes -7. Next steps -8. Appendix - - Notes on RL and deep RL - - How to set up video playback in your Jupyter notebook ---- - -### A note on RL and deep RL - -In [_RL_](https://en.wikipedia.org/wiki/Reinforcement_learning), your agent learns from trial and error by interacting with an environment using a so-called policy to gain experience. After taking one action, the agent receives information about its reward (which it may or may not get) and the next observation of the environment. It can then proceed to take another action. This happens over a number of episodes and/or until the task is deemed to be complete. - -The agent's policy works by "mapping" the agent's observations to its actions — that is, assigning a presentation of what the agent observes with required actions. The overall goal is usually to optimize the agent's policy such that it maximizes the expected rewards from each observation. - -For detailed information about RL, there is an [introductory book](https://web.archive.org/web/20050806080008/http://www.cs.ualberta.ca/~sutton/book/the-book.html) by Richard Sutton and Andrew Barton. - -Check out the Appendix at the end of the tutorial for more information. - -### Deep RL glossary - -Below is a concise glossary of deep RL terms you may find useful for the remaining part of the tutorial: - -- In a finite-horizon world, such as a game of Pong, the learning agent can explore (and exploit) the _environment_ over an _episode_. It usually takes many episodes for the agent to learn. -- The agent interacts with the _environment_ using _actions_. -- After taking an action, the agent receives some feedback through a _reward_ (if there is one), depending on which action it takes and the _state_ it is in. The state contains information about the environment. -- The agent's _observation_ is a partial observation of the state — this is the term this tutorial prefers (instead of _state_). -- The agent can choose an action based on cumulative _rewards_ (also known as the _value function_) and the _policy_. The _cumulative reward function_ estimates the quality of the observations the agent visits using its _policy_. -- The _policy_ (defined by a neural network) outputs action choices (as (log) probabilities) that should maximize the cumulative rewards from the state the agent is in. -- The _expected return from an observation_, conditional to the action, is called the _action-value_ function. To provide more weight to shorter-term rewards versus the longer-term ones, you usually use a _discount factor_ (often a floating point number between 0.9 and 0.99). -- The sequence of actions and states (observations) during each policy "run" by the agent is sometimes referred to as a _trajectory_ — such a sequence yields _rewards_. - -You will train your Pong agent through an "on-policy" method using policy gradients — it's an algorithm belonging to a family of _policy-based_ methods. Policy gradient methods typically update the parameters of the policy with respect to the long-term cumulative reward using [_gradient descent_](https://en.wikipedia.org/wiki/Gradient_descent) that is widely used in machine learning. And, since the goal is to maximize the function (the rewards), not minimize it, the process is also called _gradient ascent_. In other words, you use a policy for the agent to take actions and the objective is to maximize the rewards, which you do by computing the gradients and use them to update the parameters in the policy (neural) network. - -## Set up Pong - -**1.** First, you should install OpenAI Gym (using `pip install gym[atari]` - this package is currently not available on conda), and import NumPy, Gym and the necessary modules: - -```python -import numpy as np -import gym -``` - -Gym can monitor and save the output using the `Monitor` wrapper: - -```python -from gym import wrappers -from gym.wrappers import Monitor -``` - -**2.** Instantiate a Gym environment for the game of Pong: - -```python -env = gym.make("Pong-v0") -``` - -**3.** Let's review which actions are available in the `Pong-v0` environment: - -```python -print(env.action_space) -``` - -```python -print(env.get_action_meanings()) -``` - -There are 6 actions. However, `LEFTFIRE` is actually `LEFT`, `RIGHTFIRE` — `RIGHT`, and `NOOP` — `FIRE`. - -For simplicity, your policy network will have one output — a (log) probability for "moving up" (indexed at `2` or `RIGHT`). The other available action will be indexed at 3 ("move down" or `LEFT`). - -**4.** Gym can save videos of the agent's learning in an MP4 format — wrap `Monitor()` around the environment by running the following: - -```python -env = Monitor(env, "./video", force=True) -``` - -While you can perform all kinds of RL experiments in a Jupyter notebook, rendering images or videos of a Gym environment to visualize how your agent plays the game of Pong after training can be rather challenging. If you want to set up video playback in a notebook, you can find the details in the Appendix at the end of this tutorial. - -## Preprocess frames (the observation) - -In this section you will set up a function to preprocess the input data (game observation) to make it digestible for the neural network, which can only work with inputs that are in a form of tensors (multidimensional arrays) of floating-point type. - -Your agent will use the frames from the Pong game — pixels from screen frames — as input-observations for the policy network. The game observation tells the agent about where the ball is before it is fed (with a forward pass) into the neural network (the policy). This is similar to DeepMind's [DQN](https://deepmind.com/research/open-source/dqn) method (which is further discussed in the Appendix). - -Pong screen frames are 210x160 pixels over 3 color dimensions (red, green and blue). The arrays are encoded with `uint8` (or 8-bit integers), and these observations are stored on a Gym Box instance. - -**1.** Check the Pong's observations: - -```python -print(env.observation_space) -``` - -In Gym, the agent's actions and observations can be part of the `Box` (n-dimensional) or `Discrete` (fixed-range integers) classes. - -**2.** You can view a random observation — one frame — by: - - 1) Setting the random `seed` before initialization (optional). - - 2) Calling Gym's `reset()` to reset the environment, which returns an initial observation. - - 3) Using Matplotlib to display the `render`ed observation. - -(You can refer to the OpenAI Gym core [API](https://github.com/openai/gym/blob/master/gym/core.py) for more information about Gym's core classes and methods.) - -```python -import matplotlib.pyplot as plt - -env.seed(42) -env.reset() -random_frame = env.render(mode="rgb_array") -print(random_frame.shape) -plt.imshow(random_frame) -``` - -To feed the observations into the policy (neural) network, you need to convert them into 1D grayscale vectors with 6,400 (80x80x1) floating point arrays. (During training, you will use NumPy's [`np.ravel()`](https://numpy.org/doc/stable/reference/generated/numpy.ravel.html) function to flatten these arrays.) - -**3.** Set up a helper function for frame (observation) preprocessing: - -```python -def frame_preprocessing(observation_frame): - # Crop the frame. - observation_frame = observation_frame[35:195] - # Downsample the frame by a factor of 2. - observation_frame = observation_frame[::2, ::2, 0] - # Remove the background and apply other enhancements. - observation_frame[observation_frame == 144] = 0 # Erase the background (type 1). - observation_frame[observation_frame == 109] = 0 # Erase the background (type 2). - observation_frame[observation_frame != 0] = 1 # Set the items (rackets, ball) to 1. - # Return the preprocessed frame as a 1D floating-point array. - return observation_frame.astype(float) -``` - -**4.** Preprocess the random frame from earlier to test the function — the input for the policy network is an 80x80 1D image: - -```python -preprocessed_random_frame = frame_preprocessing(random_frame) -plt.imshow(preprocessed_random_frame, cmap="gray") -print(preprocessed_random_frame.shape) -``` - -## Create the policy (the neural network) and the forward pass - -Next, you will define the policy as a simple feedforward network that uses a game observation as an input and outputs an action log probability: - -- For the _input_, it will use the Pong video game frames — the preprocessed 1D vectors with 6,400 (80x80) floating point arrays. -- The _hidden layer_ will compute the weighted sum of inputs using NumPy's dot product function [`np.dot()`](https://numpy.org/doc/stable/reference/generated/numpy.dot.html) for the arrays and then apply a _non-linear activation function_, such as [ReLU](https://en.wikipedia.org/wiki/Rectifier_(neural_networks)). -- Then, the _output layer_ will perform the matrix-multiplication again of weight parameters and the hidden layer's output (with [`np.dot()`](https://numpy.org/doc/stable/reference/generated/numpy.dot.html)), and send that information through a [softmax](https://en.wikipedia.org/wiki/Softmax_function) _activation function_. -- In the end, the policy network will output one action log probability (given that observation) for the agent — the probability for Pong action indexed in the environment at 2 ("moving the racket up"). - -**1.** Let's instantiate certain parameters for the input, hidden, and output layers, and start setting up the network model. - -Start by creating a random number generator instance for the experiment -(seeded for reproducibility): - -```python -rng = np.random.default_rng(seed=12288743) -``` - -Then: - - - Set the input (observation) dimensionality - your preprocessed screen frames: - -```python -D = 80 * 80 -``` - - - Set the number of hidden layer neurons. - -```python -H = 200 -``` - - - Instantiate your policy (neural) network model as an empty dictionary. - -```python -model = {} -``` - -In a neural network, _weights_ are important adjustable parameters that the network fine-tunes by forward and backward propagating the data. - -**2.** Using a technique called [Xavier initialization](https://www.deeplearning.ai/ai-notes/initialization/#IV), set up the network model's initial weights with NumPy's [`Generator.standard_normal()`](https://numpy.org/doc/stable/reference/random/generated/numpy.random.Generator.standard_normal.html) that returns random numbers over a standard Normal distribution, as well as [`np.sqrt()`](https://numpy.org/doc/stable/reference/generated/numpy.sqrt.html?highlight=numpy.sqrt#numpy.sqrt): - -```python -model["W1"] = rng.standard_normal(size=(H, D)) / np.sqrt(D) -model["W2"] = rng.standard_normal(size=H) / np.sqrt(H) -``` - -**3.** Your policy network starts by randomly initializing the weights and feeds the input data (frames) forward from the input layer through a hidden layer to the output layers. This process is called the _forward pass_ or _forward propagation_, and is outlined in the function `policy_forward()`: - -```python -def policy_forward(x, model): - # Matrix-multiply the weights by the input in the one and only hidden layer. - h = np.dot(model["W1"], x) - # Apply non-linearity with ReLU. - h[h < 0] = 0 - # Calculate the "dot" product in the outer layer. - # The input for the sigmoid function is called logit. - logit = np.dot(model["W2"], h) - # Apply the sigmoid function (non-linear activation). - p = sigmoid(logit) - # Return a log probability for the action 2 ("move up") - # and the hidden "state" that you need for backpropagation. - return p, h -``` - -Note that there are two _activation functions_ for determining non-linear relationships between inputs and outputs. These [non-linear functions](https://en.wikipedia.org/wiki/Activation_function) are applied to the output of the layers: - -- [Rectified linear unit (ReLU)](https://en.wikipedia.org/wiki/Rectifier_(neural_networks)): defined as `h[h<0] = 0` above. It returns 0 for negative inputs and the same value if it's positive. -- [Sigmoid](https://en.wikipedia.org/wiki/Sigmoid_function): defined below as `sigmoid()`. It "wraps" the last layer's output and returns an action log probability in the (0, 1) range. - -**4.** Define the sigmoid function separately with NumPy's [`np.exp()`](https://numpy.org/doc/stable/reference/generated/numpy.exp.html?highlight=numpy.exp#numpy.exp) for computing exponentials: - -```python -def sigmoid(x): - return 1.0 / (1.0 + np.exp(-x)) -``` - -## Set up the update step (backpropagation) - -During learning in your deep RL algorithm, you use the action log probabilities (given an observation) and the discounted returns (for example, +1 or -1 in Pong) and perform the _backward pass_ or _backpropagation_ to update the parameters — the policy network's weights. - -**1.** Let's define the backward pass function (`policy_backward()`) with the help of NumPy's modules for array multiplication — [`np.dot()`](https://numpy.org/doc/stable/reference/generated/numpy.dot.html?highlight=numpy.dot#numpy.dot) (matrix multiplication), [`np.outer()`](https://numpy.org/doc/stable/reference/generated/numpy.outer.html) (outer product computation), and [`np.ravel()`](https://numpy.org/doc/stable/reference/generated/numpy.ravel.html) (to flatten arrays into 1D arrays): - -```python -def policy_backward(eph, epdlogp, model): - dW2 = np.dot(eph.T, epdlogp).ravel() - dh = np.outer(epdlogp, model["W2"]) - dh[eph <= 0] = 0 - dW1 = np.dot(dh.T, epx) - # Return new "optimized" weights for the policy network. - return {"W1": dW1, "W2": dW2} -``` - -Using the intermediate hidden "states" of the network (`eph`) and the gradients of action log probabilities (`epdlogp`) for an episode, the `policy_backward` function propagates the gradients back through the policy network and update the weights. - -**2.** When applying backpropagation during agent training, you will need to save several variables for each episode. Let's instantiate empty lists to store them: - -```python -# All preprocessed observations for the episode. -xs = [] -# All hidden "states" (from the network) for the episode. -hs = [] -# All gradients of probability actions -# (with respect to observations) for the episode. -dlogps = [] -# All rewards for the episode. -drs = [] -``` - -You will reset these variables manually at the end of each episode during training after they are "full" and reshape with NumPy's [`np.vstack()`](https://numpy.org/doc/stable/reference/generated/numpy.vstack.html). This is demonstrated in the training stage towards the end of the tutorial. - -**3.** Next, to perform a gradient ascent when optimizing the agent's policy, it is common to use deep learning _optimizers_ (you're performing optimization with gradients). In this example, you'll use [RMSProp](https://en.wikipedia.org/wiki/Stochastic_gradient_descent#RMSProp) — an adaptive optimization [method](http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf). Let's set a discounting factor — a decay rate — for the optimizer: - -```python -decay_rate = 0.99 -``` - -**4.** You will also need to store the gradients (with the help of NumPy's [`np.zeros_like()`](https://numpy.org/doc/stable/reference/generated/numpy.zeros_like.html)) for the optimization step during training: - -- First, save the update buffers that add up gradients over a batch: - -```python -grad_buffer = {k: np.zeros_like(v) for k, v in model.items()} -``` - -- Second, store the RMSProp memory for the optimizer for gradient ascent: - -```python -rmsprop_cache = {k: np.zeros_like(v) for k, v in model.items()} -``` - -## Define the discounted rewards (expected return) function - -In this section, you will set up a function for computing discounted rewards (`discount_rewards()`) — the expected return from an observation — that uses a 1D array of rewards as inputs (with the help of NumPy's [`np.zeros_like()`](https://numpy.org/doc/stable/reference/generated/numpy.zeros_like.html)) function. - -To provide more weight to shorter-term rewards over longer-term ones, you will use a _discount factor_ (gamma) that is often a floating-point number between 0.9 and 0.99. - -```python -gamma = 0.99 - - -def discount_rewards(r, gamma): - discounted_r = np.zeros_like(r) - running_add = 0 - # From the last reward to the first... - for t in reversed(range(0, r.size)): - # ...reset the reward sum - if r[t] != 0: - running_add = 0 - # ...compute the discounted reward - running_add = running_add * gamma + r[t] - discounted_r[t] = running_add - return discounted_r -``` - -## Train the agent for a number of episodes - -This section covers how to set up the training process during which your agent will be learning to play Pong using its policy. - -The pseudocode for the policy gradient method for Pong: - -- Instantiate the policy — your neural network — and randomly initialize the weights in the policy network. -- Initialize a random observation. -- Randomly initialize the weights in the policy network. -- Repeat over a number of episodes: - - - Input an observation into the policy network and output action probabilities for the agent (forward propagation). - - The agent takes an action for each observation, observes the received rewards and collects trajectories (over a predefined number of episodes or batch size) of state-action experiences. - - Compute the [cross-entropy](https://en.wikipedia.org/wiki/Cross_entropy#Cross-entropy_loss_function_and_logistic_regression) (with a positive sign, since you need to maximize the rewards and not minimize the loss). - - For every batch of episodes: - - - Calculate the gradients of your action log probabilities using the cross-entropy. - - Compute the cumulative return and, to provide more weight to shorter-term rewards versus the longer-term ones, use a discount factor discount. - - Multiply the gradients of the action log probabilities by the discounted rewards (the "advantage"). - - Perform gradient ascent (backpropagation) to optimize the policy network's parameters (its weights). - - - Maximize the probability of actions that lead to high rewards. - -![Diagram showing operations detailed in this tutorial](_static/tutorial-deep-reinforcement-learning-with-pong-from-pixels.png) - -You can stop the training at any time or/and check saved MP4 videos of saved plays on your disk in the `/video` directory. You can set the maximum number of episodes that is more appropriate for your setup. - -**1.** For demo purposes, let's limit the number of episodes for training to 3. If you are using hardware acceleration (CPUs and GPUs), you can increase the number to 1,000 or beyond. For comparison, Andrej Karpathy's original experiment took about 8,000 episodes. - -```python -max_episodes = 3 -``` - -**2.** Set the batch size and the learning rate values: -- The _batch size_ dictates how often (in episodes) the model performs a parameter update. It is the number of times your agent can collect the state-action trajectories. At the end of the collection, you can perform the maximization of action-probability multiples. -- The [_learning rate_](https://en.wikipedia.org/wiki/Learning_rate) helps limit the magnitude of weight updates to prevent them from overcorrecting. - -```python -batch_size = 3 -learning_rate = 1e-4 -``` - -**3.** Set the game rendering default variable for Gym's `render` method (it is used to display the observation and is optional but can be useful during debugging): - -```python -render = False -``` - -**4.** Set the agent's initial (random) observation by calling `reset()`: - -```python -observation = env.reset() -``` - -**5.** Initialize the previous observation: - -```python -prev_x = None -``` - -**6.** Initialize the reward variables and the episode count: - -```python -running_reward = None -reward_sum = 0 -episode_number = 0 -``` - -**7.** To simulate motion between the frames, set the single input frame (`x`) for the policy network as the difference between the current and previous preprocessed frames: - -```python -def update_input(prev_x, cur_x, D): - if prev_x is not None: - x = cur_x - prev_x - else: - x = np.zeros(D) - return x -``` - -**8.** Finally, start the training loop, using the functions you have predefined: - -```python -:tags: [output_scroll] - -while episode_number < max_episodes: - # (For rendering.) - if render: - env.render() - - # 1. Preprocess the observation (a game frame) and flatten with NumPy's `ravel()`. - cur_x = frame_preprocessing(observation).ravel() - - # 2. Instantiate the observation for the policy network - x = update_input(prev_x, cur_x, D) - prev_x = cur_x - - # 3. Perform the forward pass through the policy network using the observations - # (preprocessed frames as inputs) and store the action log probabilities - # and hidden "states" (for backpropagation) during the course of each episode. - aprob, h = policy_forward(x, model) - # 4. Let the action indexed at `2` ("move up") be that probability - # if it's higher than a randomly sampled value - # or use action `3` ("move down") otherwise. - action = 2 if rng.uniform() < aprob else 3 - - # 5. Cache the observations and hidden "states" (from the network) - # in separate variables for backpropagation. - xs.append(x) - hs.append(h) - - # 6. Compute the gradients of action log probabilities: - # - If the action was to "move up" (index `2`): - y = 1 if action == 2 else 0 - # - The cross-entropy: - # `y*log(aprob) + (1 - y)*log(1-aprob)` - # or `log(aprob)` if y = 1, else: `log(1 - aprob)`. - # (Recall: you used the sigmoid function (`1/(1+np.exp(-x)`) to output - # `aprob` action probabilities.) - # - Then the gradient: `y - aprob`. - # 7. Append the gradients of your action log probabilities. - dlogps.append(y - aprob) - # 8. Take an action and update the parameters with Gym's `step()` - # function; obtain a new observation. - observation, reward, done, info = env.step(action) - # 9. Update the total sum of rewards. - reward_sum += reward - # 10. Append the reward for the previous action. - drs.append(reward) - - # After an episode is finished: - if done: - episode_number += 1 - # 11. Collect and reshape stored values with `np.vstack()` of: - # - Observation frames (inputs), - epx = np.vstack(xs) - # - hidden "states" (from the network), - eph = np.vstack(hs) - # - gradients of action log probabilities, - epdlogp = np.vstack(dlogps) - # - and received rewards for the past episode. - epr = np.vstack(drs) - - # 12. Reset the stored variables for the new episode: - xs = [] - hs = [] - dlogps = [] - drs = [] - - # 13. Discount the rewards for the past episode using the helper - # function you defined earlier... - discounted_epr = discount_rewards(epr, gamma) - # ...and normalize them because they have high variance - # (this is explained below.) - discounted_epr -= np.mean(discounted_epr) - discounted_epr /= np.std(discounted_epr) - - # 14. Multiply the discounted rewards by the gradients of the action - # log probabilities (the "advantage"). - epdlogp *= discounted_epr - # 15. Use the gradients to perform backpropagation and gradient ascent. - grad = policy_backward(eph, epdlogp, model) - # 16. Save the policy gradients in a buffer. - for k in model: - grad_buffer[k] += grad[k] - # 17. Use the RMSProp optimizer to perform the policy network - # parameter (weight) update at every batch size - # (by default: every 10 episodes). - if episode_number % batch_size == 0: - for k, v in model.items(): - # The gradient. - g = grad_buffer[k] - # Use the RMSProp discounting factor. - rmsprop_cache[k] = ( - decay_rate * rmsprop_cache[k] + (1 - decay_rate) * g ** 2 - ) - # Update the policy network with a learning rate - # and the RMSProp optimizer using gradient ascent - # (hence, there's no negative sign) - model[k] += learning_rate * g / (np.sqrt(rmsprop_cache[k]) + 1e-5) - # Reset the gradient buffer at the end. - grad_buffer[k] = np.zeros_like(v) - - # 18. Measure the total discounted reward. - running_reward = ( - reward_sum - if running_reward is None - else running_reward * 0.99 + reward_sum * 0.01 - ) - print( - "Resetting the Pong environment. Episode total reward: {} Running mean: {}".format( - reward_sum, running_reward - ) - ) - - # 19. Set the agent's initial observation by calling Gym's `reset()` function - # for the next episode and setting the reward sum back to 0. - reward_sum = 0 - observation = env.reset() - prev_x = None - - # 20. Display the output during training. - if reward != 0: - print( - "Episode {}: Game finished. Reward: {}...".format(episode_number, reward) - + ("" if reward == -1 else " POSITIVE REWARD!") - ) -``` - -A few notes: - -- If you have previously run an experiment and want to repeat it, your `Monitor` instance may still be running, which may throw an error the next time you try to traini the agent. Therefore, you should first shut down `Monitor` by calling `env.close()` by uncommenting and running the cell below: - -```python -# env.close() -``` - -- In Pong, if a player doesn't hit the ball back, they receive a negative reward (-1) and the other player gets a +1 reward. The rewards that the agent receives by playing Pong have a significant variance. Therefore, it's best practice to normalize them with the same mean (using [`np.mean()`](https://numpy.org/doc/stable/reference/generated/numpy.mean.html)) and standard deviation (using NumPy's [`np.std()`](https://numpy.org/doc/stable/reference/generated/numpy.std.html?highlight=std)). - -- When using only NumPy, the deep RL training process, including backpropagation, spans several lines of code that may appear quite long. One of the main reasons for this is you're not using a deep learning framework with an automatic differentiation library that usually simplifies such experiments. This tutorial shows how to perform everything from scratch but you can also use one of many Python-based frameworks with "autodiff" and "autograd", which you will learn about at the end of the tutorial. - -## Next steps - -You may notice that training an RL agent takes a long time if you increase the number of episodes from 100 to 500 or 1,000+, depending on the hardware — CPUs and GPUs — you are using for this task. - -Policy gradient methods can learn a task if you give them a lot of time, and optimization in RL is a challenging problem. Training agents to learn to play Pong or any other task can be sample-inefficient and require a lot of episodes. You may also notice in your training output that even after hundreds of episodes, the rewards may have high variance. - -In addition, like in many deep learning-based algorithms, you should take into account a large amount of parameters that your policy has to learn. In Pong, this number adds up to 1 million or more with 200 nodes in the hidden layer of the network and the input dimension being of size 6,400 (80x80). Therefore, adding more CPUs and GPUs to assist with training can always be an option. - -You can use a much more advanced policy gradient-based algorithm that can help speed up training, improve the sensitivity to parameters, and resolve other issues. For example, there are "self-play" methods, such as [Proximal Policy Optimization (PPO)](https://arxiv.org/pdf/1707.06347) developed by [John Schulman](http://joschu.net) et al in 2017, which were [used](https://openai.com/blog/openai-five/#rapid) to train the [OpenAI Five](https://arxiv.org/pdf/1912.06680.pdf) agent over 10 months to play Dota 2 at a competitive level. Of course, if you apply these methods to smaller Gym environments, it should take hours, not months to train. - -In general, there are many RL challenges and possible solutions and you can explore some of them in [Reinforcement learning, fast and slow](https://static1.squarespace.com/static/555aab07e4b03e184ddaf731/t/5f5245cb100273193b14548a/1599227416572/TICS__RL_Fast_and_Slow_accepted.pdf) by [Matthew Botvinick](https://hai.stanford.edu/people/matthew-botvinick), Sam Ritter, [Jane X. Wang](http://www.janexwang.com), Zeb Kurth-Nelson, [Charles Blundell](http://www.gatsby.ucl.ac.uk/~ucgtcbl/), and [Demis Hassabis](https://en.wikipedia.org/wiki/Demis_Hassabis) (2019). - ---- - -If you want to learn more about deep RL, you should check out the following free educational material: - -- [Spinning Up in Deep RL](https://openai.com/blog/spinning-up-in-deep-rl/): developed by OpenAI. -- Deep RL lectures taught by practitioners at [DeepMind](https://www.youtube.com/c/DeepMind/videos) and [UC Berkeley](https://www.youtube.com/channel/UC4e_-TvgALrwE1dUPvF_UTQ/videos). -- RL [lectures](https://www.davidsilver.uk/teaching/) taught by [David Silver](https://www.davidsilver.uk) (DeepMind, UCL). - -Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as [PyTorch](https://pytorch.org/), [JAX](https://github.com/google/jax), [TensorFlow](https://www.tensorflow.org/guide/tf_numpy) or [MXNet](https://mxnet.apache.org) — that provide NumPy-like APIs, have built-in [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) and GPU support, and are designed for high-performance numerical computing and machine learning. - - -## Appendix - -### Notes on RL and deep RL - -- In [supervised](https://en.wikipedia.org/wiki/Supervised_learning) deep learning for tasks, such as image recognition, language translation, or text classification, you're more likely to use a lot of labeled data. However, in RL, agents typically don't receive direct explicit feedback indicating correct or wrong actions — they rely on other signals, such as rewards. - -- _Deep RL_ combines RL with [deep learning](http://www.cs.toronto.edu/~hinton/absps/NatureDeepReview.pdf). The field had its first major success in more complex environments, such as video games, in 2013 — a year after the [AlexNet](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf) breakthrough in computer vision. Volodymyr Mnih and colleagues at DeepMind published a research paper called [Playing Atari with deep reinforcement learning](https://arxiv.org/abs/1312.5602) (and [updated](https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf) in 2015) that showed that they were able to train an agent that could play several classic games from the Arcade Learning Environment at a human-level. Their RL algorithm — called a deep Q-network (DQN) — used [convolutional layers](https://en.wikipedia.org/wiki/Convolutional_neural_network) in a neural network that approximated [Q learning](https://en.wikipedia.org/wiki/Q-learning) and used [experience replay](https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf). - -- Unlike the simple policy gradient method that you used in this example, DQN uses a type of "off-policy" _value-based_ method (that approximates Q learning), while the original [AlphaGo](https://www.nature.com/articles/nature24270.epdf?author_access_token=VJXbVjaSHxFoctQQ4p2k4tRgN0jAjWel9jnR3ZoTv0PVW4gB86EEpGqTRDtpIz-2rmo8-KG06gqVobU5NSCFeHILHcVFUeMsbvwS-lxjqQGg98faovwjxeTUgZAUMnRQ) uses policy gradients and [Monte Carlo tree search](https://en.wikipedia.org/wiki/Monte_Carlo_tree_search). - -- Policy gradients _with function approximation_, such as neural networks, were [written about](https://papers.nips.cc/paper/1713-policy-gradient-methods-for-reinforcement-learning-with-function-approximation.pdf) in 2000 by Richard Sutton et al. They were influenced by a number of previous works, including statistical gradient-following algorithms, such as [REINFORCE](https://www.semanticscholar.org/paper/Simple-statistical-gradient-following-algorithms-Williams/4c915c1eecb217c123a36dc6d3ce52d12c742614) (Ronald Williams, 1992), as well as [backpropagation](http://www.cs.toronto.edu/~hinton/absps/naturebp.pdf) (Geoffrey Hinton, 1986), which helps deep learning algorithms learn. RL with neural-network function approximation were introduced in the 1990s in research by Gerald Tesauro ([Temporal difference learning and td-gammon](https://dl.acm.org/doi/10.1145/203330.203343), 1995), who worked with IBM on an agent that learned to [play backgammon](https://en.wikipedia.org/wiki/TD-Gammon) in 1992, and Long-Ji Lin ([Reinforcement learning for robots using neural networks](https://dl.acm.org/doi/book/10.5555/168871), 1993). - -- Since 2013, researchers have come up with many notable approaches for learning to solve complex tasks using deep RL, such as [AlphaGo](https://www.nature.com/articles/nature24270.epdf?author_access_token=VJXbVjaSHxFoctQQ4p2k4tRgN0jAjWel9jnR3ZoTv0PVW4gB86EEpGqTRDtpIz-2rmo8-KG06gqVobU5NSCFeHILHcVFUeMsbvwS-lxjqQGg98faovwjxeTUgZAUMnRQ) for the game of Go (David Silver et al, 2016), [AlphaZero](http://science.sciencemag.org/cgi/content/full/362/6419/1140?ijkey=XGd77kI6W4rSc&keytype=ref&siteid=sci) that mastered Go, Chess, and Shogi with self-play (David Silver et al, 2017-2018), [OpenAI Five](https://arxiv.org/pdf/1912.06680.pdf) for Dota 2 with [self-play](https://openai.com/blog/competitive-self-play/) (OpenAI, 2019), and [AlphaStar](https://deepmind.com/blog/alphastar-mastering-real-time-strategy-game-starcraft-ii/) for StarCraft 2 that used an [actor-critic](https://arxiv.org/pdf/1802.01561.pdf) algorithm with [experience replay](https://link.springer.com/content/pdf/10.1023%2FA%3A1022628806385.pdf), [self-imitation learning](http://proceedings.mlr.press/v80/oh18b/oh18b.pdf), and [policy distillation](https://arxiv.org/pdf/1511.06295.pdf) (Oriol Vinyals et al, 2019). In addition, there have been other experiments, such as deep RL for [Battlefield 1](https://www.ea.com/seed/news/self-learning-agents-play-bf1) by engineers at Electronic Arts/DICE. - -- One of the reasons why video games are popular in deep RL research is that, unlike real-world experiments, such as RL with [remote-controlled helicopters](http://heli.stanford.edu/papers/nips06-aerobatichelicopter.pdf) ([Pieter Abbeel](https://www2.eecs.berkeley.edu/Faculty/Homepages/abbeel.html) et al, 2006), virtual simulations can offer safer testing environments. - -- If you're interested in learning about the implications of deep RL on other fields, such as neuroscience, you can refer to a [paper](https://arxiv.org/pdf/2007.03750.pdf) by [Matthew Botvinick](https://www.youtube.com/watch?v=b0LddBiF5jM) et al (2020). - -### How to set up video playback in your Jupyter notebook - -- If you're using [**Binder**](https://mybinder.org) — a free Jupyter notebook-based tool — you can set up the Docker image and add `freeglut3-dev`, `xvfb`, and `x11-utils` to the `apt.txt` configuration file to install the initial dependencies. Then, to `binder/environment.yml` under `channels`, add `gym`, `pyvirtualdisplay` and anything else you may need, such as `python=3.7`, `pip`, and `jupyterlab`. Check the following [post](https://towardsdatascience.com/rendering-openai-gym-envs-on-binder-and-google-colab-536f99391cc7) for more information. - -- If you're using [**Google Colaboratory**](https://colab.research.google.com/notebooks/intro.ipynb) (another free Jupyter notebook-based tool), you can enable video playback of the game environments installing and setting up [X virtual frame buffer](https://en.wikipedia.org/wiki/Xvfb)/[Xvfb](https://www.x.org/releases/X11R7.6/doc/man/man1/Xvfb.1.xhtml), [X11](https://en.wikipedia.org/wiki/X_Window_System), [FFmpeg](https://ffmpeg.org), [PyVirtualDisplay](https://github.com/ponty/PyVirtualDisplay), [PyOpenGL](http://pyopengl.sourceforge.net), and other dependencies, as described further below. - -1. If you're using Google Colaboratory, run the following commands in the notebook cells to help with video playback: - - ```shell - # Install Xvfb and X11 dependencies. - !apt-get install -y xvfb x11-utils > /dev/null 2>&1 - # To work with videos, install FFmpeg. - !apt-get install -y ffmpeg > /dev/null 2>&1 - # Install PyVirtualDisplay for visual feedback and other libraries/dependencies. - !pip install pyvirtualdisplay PyOpenGL PyOpenGL-accelerate > /dev/null 2>&1 - ``` - -2. Then, add this Python code: - - ``` - # Import the virtual display module. - from pyvirtualdisplay import Display - # Import ipythondisplay and HTML from IPython for image and video rendering. - from IPython import display as ipythondisplay - from IPython.display import HTML - - # Initialize the virtual buffer at 400x300 (adjustable size). - # With Xvfb, you should set `visible=False`. - display = Display(visible=False, size=(400, 300)) - display.start() - - # Check that no display is present. - # If no displays are present, the expected output is `:0`. - !echo $DISPLAY - - # Define a helper function to display videos in Jupyter notebooks:. - # (Source: https://star-ai.github.io/Rendering-OpenAi-Gym-in-Colaboratory/) - - import sys - import math - import glob - import io - import base64 - - def show_any_video(mp4video=0): - mp4list = glob.glob('video/*.mp4') - if len(mp4list) > 0: - mp4 = mp4list[mp4video] - video = io.open(mp4, 'r+b').read() - encoded = base64.b64encode(video) - ipythondisplay.display(HTML(data=''''''.format(encoded.decode('ascii')))) - - else: - print('Could not find the video!') - - ``` - -- If you want to view the last (very quick) gameplay inside a Jupyter notebook and implemented the `show_any_video()` function earlier, run this inside a cell: - - ```py - show_any_video(-1) - ``` - -- If you're following the instructions in this tutorial in a local environment on Linux or macOS, you can add most of the code into one **Python (`.py`)** file. Then, you can run your Gym experiment through `python your-code.py` in your terminal. To enable rendering, you can use the command-line interface by following the [official OpenAI Gym documentation](https://github.com/openai/gym#rendering-on-a-server) (make sure you have Gym and Xvfb installed, as described in the guide). diff --git a/content/tutorial-ma.md b/content/tutorial-ma.md index f4aa9a98..fcb69eb2 100644 --- a/content/tutorial-ma.md +++ b/content/tutorial-ma.md @@ -92,7 +92,7 @@ rows of this file, since they contain other data we are not interested in. Separ # Read just the dates for columns 4-18 from the first row dates = np.genfromtxt( filename, - dtype=np.unicode_, + dtype=np.str_, delimiter=",", max_rows=1, usecols=range(4, 18), @@ -102,7 +102,7 @@ dates = np.genfromtxt( # columns, skipping the first six rows locations = np.genfromtxt( filename, - dtype=np.unicode_, + dtype=np.str_, delimiter=",", skip_header=6, usecols=(0, 1), @@ -119,7 +119,7 @@ nbcases = np.genfromtxt( ) ``` -Included in the `numpy.genfromtxt` function call, we have selected the [numpy.dtype](https://numpy.org/devdocs/reference/generated/numpy.dtype.html#numpy.dtype) for each subset of the data (either an integer - `numpy.int_` - or a string of characters - `numpy.unicode_`). We have also used the `encoding` argument to select `utf-8-sig` as the encoding for the file (read more about encoding in the [official Python documentation](https://docs.python.org/3/library/codecs.html#encodings-and-unicode). You can read more about the `numpy.genfromtxt` function from the [Reference Documentation](https://numpy.org/devdocs/reference/generated/numpy.genfromtxt.html#numpy.genfromtxt) or from the [Basic IO tutorial](https://numpy.org/devdocs/user/basics.io.genfromtxt.html). +Included in the `numpy.genfromtxt` function call, we have selected the [numpy.dtype](https://numpy.org/devdocs/reference/generated/numpy.dtype.html#numpy.dtype) for each subset of the data (either an integer - `numpy.int_` - or a string of characters - `numpy.str_`). We have also used the `encoding` argument to select `utf-8-sig` as the encoding for the file (read more about encoding in the [official Python documentation](https://docs.python.org/3/library/codecs.html#encodings-and-unicode). You can read more about the `numpy.genfromtxt` function from the [Reference Documentation](https://numpy.org/devdocs/reference/generated/numpy.genfromtxt.html#numpy.genfromtxt) or from the [Basic IO tutorial](https://numpy.org/devdocs/user/basics.io.genfromtxt.html). +++ @@ -264,14 +264,15 @@ Now, if we want to create a very simple approximation for this data, we should t dates[~china_total.mask] ``` -Finally, we can use the [numpy.polyfit](https://numpy.org/devdocs/reference/generated/numpy.polyfit.html#numpy.polyfit) and [numpy.polyval](https://numpy.org/devdocs/reference/generated/numpy.polyval.html#numpy.polyval) functions to create a cubic polynomial that fits the data as best as possible: +Finally, we can use the +[fitting functionality of the numpy.polynomial](https://numpy.org/doc/stable/reference/generated/numpy.polynomial.polynomial.Polynomial.fit.html) +package to create a cubic polynomial model that fits the data as best as possible: ```{code-cell} t = np.arange(len(china_total)) -params = np.polyfit(t[~china_total.mask], valid, 3) -cubic_fit = np.polyval(params, t) +model = np.polynomial.Polynomial.fit(t[~china_total.mask], valid, deg=3) plt.plot(t, china_total) -plt.plot(t, cubic_fit, "--") +plt.plot(t, model(t), "--") ``` This plot is not so readable since the lines seem to be over each other, so let's summarize in a more elaborate plot. We'll plot the real data when @@ -279,10 +280,10 @@ available, and show the cubic fit for unavailable data, using this fit to comput ```{code-cell} plt.plot(t, china_total) -plt.plot(t[china_total.mask], cubic_fit[china_total.mask], "--", color="orange") -plt.plot(7, np.polyval(params, 7), "r*") +plt.plot(t[china_total.mask], model(t)[china_total.mask], "--", color="orange") +plt.plot(7, model(7), "r*") plt.xticks([0, 7, 13], dates[[0, 7, 13]]) -plt.yticks([0, np.polyval(params, 7), 10000, 17500]) +plt.yticks([0, model(7), 10000, 17500]) plt.legend(["Mainland China", "Cubic estimate", "7 days after start"]) plt.title( "COVID-19 cumulative cases from Jan 21 to Feb 3 2020 - Mainland China\n" diff --git a/content/tutorial-nlp-from-scratch.md b/content/tutorial-nlp-from-scratch.md deleted file mode 100644 index 865fd1c9..00000000 --- a/content/tutorial-nlp-from-scratch.md +++ /dev/null @@ -1,1053 +0,0 @@ ---- -jupyter: - jupytext: - formats: md,ipynb - text_representation: - extension: .md - format_name: markdown - format_version: '1.3' - jupytext_version: 1.11.5 - kernelspec: - display_name: Python 3 (ipykernel) - language: python - name: python3 ---- - -# Sentiment Analysis on notable speeches of the last decade - -This tutorial demonstrates how to build a simple Long Short Term memory network (LSTM) from scratch in NumPy to perform sentiment analysis on a socially relevant and ethically acquired dataset. - -Your deep learning model (the LSTM) is a form of a Recurrent Neural Network and will learn to classify a piece of text as positive or negative from the IMDB reviews dataset. The dataset contains 50,000 movie reviews and corresponding labels. Based on the numeric representations of these reviews and their corresponding labels (supervised learning) the neural network will be trained to learn the sentiment using forward propagation and backpropagation through time since we are dealing with sequential data here. The output will be a vector containing the probabilities that the text samples are positive. - - -Today, Deep Learning is getting adopted in everyday life and now it is more important to ensure that decisions that have been taken using AI are not reflecting discriminatory behavior towards a set of populations. It is important to take fairness into consideration while consuming the output from AI. Throughout the tutorial we'll try to question all the steps in our pipeline from an ethics point of view. - - -## Prerequisites - -You are expected to be familiar with the Python programming language and array manipulation with NumPy. In addition, some understanding of Linear Algebra and Calculus is recommended. You should also be familiar with how Neural Networks work. For reference, you can visit the [Python](https://docs.python.org/dev/tutorial/index.html), [Linear algebra on n-dimensional arrays](https://numpy.org/doc/stable/user/tutorial-svd.html) and [Calculus](https://d2l.ai/chapter_appendix-mathematics-for-deep-learning/multivariable-calculus.html) tutorials. - -To get a refresher on Deep Learning basics, You should consider reading [the d2l.ai book](https://d2l.ai/chapter_recurrent-neural-networks/index.html), which is an interactive deep learning book with multi-framework code, math, and discussions. You can also go through the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) to understand how a basic neural network is implemented from scratch. - -In addition to NumPy, you will be utilizing the following Python standard modules for data loading and processing: -- [`pandas`](https://pandas.pydata.org/docs/) for handling dataframes -- [`Matplotlib`](https://matplotlib.org/) for data visualization -- [`pooch`](https://www.fatiando.org/pooch/latest/https://www.fatiando.org/pooch/latest/) to download and cache datasets - -This tutorial can be run locally in an isolated environment, such as [Virtualenv](https://virtualenv.pypa.io/en/stable/) or [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html). You can use [Jupyter Notebook or JupyterLab](https://jupyter.org/install) to run each notebook cell. - - -## Table of contents - -1. Data Collection - -2. Preprocess the datasets - -3. Build and train a LSTM network from scratch - -4. Perform sentiment analysis on collected speeches - -5. Next steps - - -## 1. Data Collection - -Before you begin there are a few pointers you should always keep in mind before choosing the data you wish to train your model on: -- **Identifying Data Bias** - Bias is an inherent component of the human thought process. Therefore data sourced from human activities reflects that bias. Some ways in which this bias tends to occur in Machine Learning datasets are: - - *Bias in historical data*: Historical data are often skewed towards, or against, particular groups. - Data can also be severely imbalanced with limited information on protected groups. - - *Bias in data collection mechanisms*: Lack of representativeness introduces inherent biases in the data collection process. - - *Bias towards observable outcomes*: In some scenarios, we have the information about True Outcomes only for a certain section of the population. In the absence of information on all outcomes, one cannot even measure fairness -- **Preserving human anonymity for sensitive data**: [Trevisan and Reilly](https://eprints.whiterose.ac.uk/91157/1/Ethical%20dilemmas.pdf) identified a list of sensitive topics that need to be handled with extra care. We present the same below along with a few additions: - - personal daily routines (including location data); - - individual details about impairment and/or medical records; - - emotional accounts of pain and chronic illness; - - financial information about income and/or welfare payments; - - discrimination and abuse episodes; - - criticism/praise of individual providers of healthcare and support services; - - suicidal thoughts; - - criticism/praise of a power structure especially if it compromises their safety; - - personally-identifying information (even if anonymized in some way) including things like fingerprints or voice. - ->While it can be difficult taking consent from so many people especially on on-line platforms, the necessity of it depends upon the sensitivity of the topics your data includes and other indicators like whether the platform the data was obtained from allows users to operate under pseudonyms. If the website has a policy that forces the use of a real name, then the users need to be asked for consent. - -In this section, you will be collecting two different datasets: the IMDb movie reviews dataset, and a collection of 10 speeches curated for this tutorial including activists from different countries around the world, different times, and different topics. The former would be used to train the deep learning model while the latter will be used to perform sentiment analysis on. - - -### Collecting the IMDb reviews dataset -IMDb Reviews Dataset is a large movie review dataset collected and prepared by Andrew L. Maas from the popular movie rating service, IMDb. The IMDb Reviews dataset is used for binary sentiment classification, whether a review is positive or negative. It contains 25,000 movie reviews for training and 25,000 for testing. All these 50,000 reviews are labeled data that may be used for supervised deep learning. For ease of reproducibility, we'll be sourcing the data from [Zenodo](https://zenodo.org/record/4117827#.YVQZ_EZBy3Ihttps://zenodo.org/record/4117827#.YVQZ_EZBy3I). - > The IMDb platform allows the usage of their public datasets for personal and non-commercial use. We did our best to ensure that these reviews do not contain any of the aforementioned sensitive topics pertaining to the reviewer. - - -### Collecting and loading the speech transcripts -We have chosen speeches by activists around the globe talking about issues like climate change, feminism, lgbtqa+ rights and racism. These were sourced from newspapers, the official website of the United Nations and the archives of established universities as cited in the table below. A CSV file was created containing the transcribed speeches, their speaker and the source the speeches were obtained from. -We made sure to include different demographics in our data and included a range of different topics, most of which focus on social and/or ethical issues. - -| Speech | Speaker | Source | -|--------------------------------------------------|-------------------------|------------------------------------------------------------| -| Barnard College Commencement | Leymah Gbowee | [Barnard College](https://barnard.edu/news/transcript-speech-nobel-peace-prize-winner-leymah-gbowee) | -| UN Speech on youth Education | Malala Yousafzai | [The Guardian](https://www.theguardian.com/commentisfree/2013/jul/12/malala-yousafzai-united-nations-education-speech-text) | -| Remarks in the UNGA on racial discrimination | Linda Thomas Greenfield | [United States mission to the United Nation](https://usun.usmission.gov/remarks-by-ambassador-linda-thomas-greenfield-at-a-un-general-assembly-commemorative-meeting-for-intl-day-for-the-elimination-of-racial-discrimination/) | -| How Dare You | Greta Thunberg | [NBC](https://www.nbcnews.com/news/world/read-greta-thunberg-s-full-speech-united-nations-climate-action-n1057861) | -| The speech that silenced the world for 5 minutes | Severn Suzuki | [Earth Charter](https://earthcharter.org/new-voices-after-26-years-of-the-girl-who-silenced-the-world-for-5-minutes/) | -| The Hope Speech | Harvey Milk | [Museum of Fine Arts, Boston](https://www.mfa.org/exhibitions/amalia-pica/transcript-harvey-milks-the-hope-speech) | -| Speech at the time to Thrive Conference | Ellen Page | [Huffpost](https://www.huffpost.com/entry/time-to-thrive_b_4794251) | -| I have a dream | Martin Luther King | [Marshall University](https://www.marshall.edu/onemarshallu/i-have-a-dream/) | - - -## 2. Preprocess the datasets ->Preprocessing data is an extremely crucial step before building any Deep learning model, however in an attempt to keep the tutorial focused on building the model, we will not dive deep into the code for preprocessing. Given below is a brief overview of all the steps we undertake to clean our data and convert it to its numeric representation. - -1. **Text Denoising** : Before converting your text into vectors, it is important to clean it and remove all unhelpful parts a.k.a the noise from your data by converting all characters to lowercase, removing html tags, brackets and stop words (words that don't add much meaning to a sentence). Without this step the dataset is often a cluster of words that the computer doesn't understand. - - -2. **Converting words to vectors** : A word embedding is a learned representation for text where words that have the same meaning have a similar representation. Individual words are represented as real-valued vectors in a predefined vector space. GloVe is an unsupervised algorithm developed by Stanford for generating word embeddings by generating global word-word co-occurence matrix from a corpus. You can download the zipped files containing the embeddings from https://nlp.stanford.edu/projects/glove/. Here you can choose any of the four options for different sizes or training datasets. We have chosen the least memory consuming embedding file. - >The GloVe word embeddings include sets that were trained on billions of tokens, some up to 840 billion tokens. These algorithms exhibit stereotypical biases, such as gender bias which can be traced back to the original training data. For example certain occupations seem to be more biased towards a particular gender, reinforcing problematic stereotypes. The nearest solution to this problem are some de-biasing algorithms as the one presented in https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1184/reports/6835575.pdf which one can use on embeddings of their choice to mitigate bias, if present. - - -You'll start with importing the necessary packages to build our Deep Learning network. - -```python -# Importing the necessary packages -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import pooch -import string -import re -import zipfile -import os - -# Creating the random instance -rng = np.random.default_rng() -``` - -Next, you'll define set of text preprocessing helper functions. - -```python -class TextPreprocess: - """Text Preprocessing for a Natural Language Processing model.""" - - def txt_to_df(self, file): - """Function to convert a txt file to pandas dataframe. - - Parameters - ---------- - file : str - Path to the txt file. - - Returns - ------- - Pandas dataframe - txt file converted to a dataframe. - - """ - with open(imdb_train, 'r') as in_file: - stripped = (line.strip() for line in in_file) - reviews = {} - for line in stripped: - lines = [splits for splits in line.split("\t") if splits != ""] - reviews[lines[1]] = float(lines[0]) - df = pd.DataFrame(reviews.items(), columns=['review', 'sentiment']) - df = df.sample(frac=1).reset_index(drop=True) - return df - - def unzipper(self, zipped, to_extract): - """Function to extract a file from a zipped folder. - - Parameters - ---------- - zipped : str - Path to the zipped folder. - - to_extract: str - Path to the file to be extracted from the zipped folder - - Returns - ------- - str - Path to the extracted file. - - """ - fh = open(zipped, 'rb') - z = zipfile.ZipFile(fh) - outdir = os.path.split(zipped)[0] - z.extract(to_extract, outdir) - fh.close() - output_file = os.path.join(outdir, to_extract) - return output_file - - def cleantext(self, df, text_column=None, - remove_stopwords=True, remove_punc=True): - """Function to clean text data. - - Parameters - ---------- - df : pandas dataframe - The dataframe housing the input data. - text_column : str - Column in dataframe whose text is to be cleaned. - remove_stopwords : bool - if True, remove stopwords from text - remove_punc : bool - if True, remove punctuation symbols from text - - Returns - ------- - Numpy array - Cleaned text. - - """ - # converting all characters to lowercase - df[text_column] = df[text_column].str.lower() - - # List of stopwords taken from https://gist.github.com/sebleier/554280 - stopwords = ["a", "about", "above", "after", "again", "against", - "all", "am", "an", "and", "any", "are", - "as", "at", "be", "because", - "been", "before", "being", "below", - "between", "both", "but", "by", "could", - "did", "do", "does", "doing", "down", "during", - "each", "few", "for", "from", "further", - "had", "has", "have", "having", "he", - "he'd", "he'll", "he's", "her", "here", - "here's", "hers", "herself", "him", - "himself", "his", "how", "how's", "i", - "i'd", "i'll", "i'm", "i've", - "if", "in", "into", - "is", "it", "it's", "its", - "itself", "let's", "me", "more", - "most", "my", "myself", "nor", "of", - "on", "once", "only", "or", - "other", "ought", "our", "ours", - "ourselves", "out", "over", "own", "same", - "she", "she'd", "she'll", "she's", "should", - "so", "some", "such", "than", "that", - "that's", "the", "their", "theirs", "them", - "themselves", "then", "there", "there's", - "these", "they", "they'd", "they'll", - "they're", "they've", "this", "those", - "through", "to", "too", "under", "until", "up", - "very", "was", "we", "we'd", "we'll", - "we're", "we've", "were", "what", - "what's", "when", "when's", - "where", "where's", - "which", "while", "who", "who's", - "whom", "why", "why's", "with", - "would", "you", "you'd", "you'll", - "you're", "you've", - "your", "yours", "yourself", "yourselves"] - - def remove_stopwords(data, column): - data[f'{column} without stopwords'] = data[column].apply( - lambda x: ' '.join([word for word in x.split() if word not in (stopwords)])) - return data - - def remove_tags(string): - result = re.sub('<*>', '', string) - return result - - # remove html tags and brackets from text - if remove_stopwords: - data_without_stopwords = remove_stopwords(df, text_column) - data_without_stopwords[f'clean_{text_column}'] = data_without_stopwords[f'{text_column} without stopwords'].apply( - lambda cw: remove_tags(cw)) - if remove_punc: - data_without_stopwords[f'clean_{text_column}'] = data_without_stopwords[f'clean_{text_column}'].str.replace( - '[{}]'.format(string.punctuation), ' ', regex=True) - - X = data_without_stopwords[f'clean_{text_column}'].to_numpy() - - return X - - - def sent_tokeniser(self, x): - """Function to split text into sentences. - - Parameters - ---------- - x : str - piece of text - - Returns - ------- - list - sentences with punctuation removed. - - """ - sentences = re.split(r'(? Since we will be performing paragraph wise sentiment analysis on each speech further ahead in the tutorial, we'll need the punctuation marks to split the text into paragraphs, hence we refrain from removing their punctuation marks at this stage - -```python -speech_data_path = 'tutorial-nlp-from-scratch/speeches.csv' -speech_df = pd.read_csv(speech_data_path) -X_pred = textproc.cleantext(speech_df, - text_column='speech', - remove_stopwords=True, - remove_punc=False) -speakers = speech_df['speaker'].to_numpy() -``` - -You will now download the `GloVe` embeddings, unzip them and build a dictionary mapping each word and word embedding. This will act as a cache for when you need to replace each word with its respective word embedding. - -```python -glove = data.fetch('glove.6B.50d.zip') -emb_path = textproc.unzipper(glove, 'glove.6B.300d.txt') -emb_matrix = textproc.loadGloveModel(emb_path) -``` - -## 3. Build the Deep Learning Model¶ - It is time to start implementing our LSTM! You will have to first familiarize yourself with some high-level concepts of the basic building blocks of a deep learning model. You can refer to the [Deep learning on MNIST from scratch tutorial](https://numpy.org/numpy-tutorials/content/tutorial-deep-learning-on-mnist.html) for the same. - -You will then learn how a Recurrent Neural Network differs from a plain Neural Network and what makes it so suitable for processing sequential data. Afterwards, you will construct the building blocks of a simple deep learning model in Python and NumPy and train it to learn to classify the sentiment of a piece of text as positive or negative with a certain level of accuracy - -### Introduction to a Long Short Term Memory Network - -In a [Multilayer perceptron](https://en.wikipedia.org/wiki/Multilayer_perceptron) (MLP), the information only moves in one direction — from the input layer, through the hidden layers, to the output layer. The information moves straight through the network and never takes the previous nodes into account at a later stage. Because it only considers the current input, the features learned are not shared across different positions of the sequence. Moreover, it cannot process sequences with varying lengths. - -Unlike an MLP, the RNN was designed to work with sequence prediction problems.RNNs introduce state variables to store past information, together with the current inputs, to determine the current outputs. Since an RNN shares the learned features with all the data points in a sequence regardless of its length, it is capable of processing sequences with varying lengths. - -The problem with an RNN however, is that it cannot retain long-term memory because the influence of a given input on the hidden layer, and therefore on the network output, either decays or blows up exponentially as it cycles around the network’s recurrent connections. This shortcoming is referred to as the vanishing gradient problem. Long Short-Term Memory (LSTM) is an RNN architecture specifically designed to address the [vanishing gradient problem](https://en.wikipedia.org/wiki/Vanishing_gradient_problem). - - -### Overview of the Model Architecture - -![Overview of the model architecture, showing a series of animated boxes. There are five identical boxes labeled A and receiving as input one of the words in the phrase "life's a box of chocolates". Each box is highlighted in turn, representing the memory blocks of the LSTM network as information passes through them, ultimately reaching a "Positive" output value.](_static/lstm.gif) - -In the above gif, the rectangles labeled $A$ are called `Cells` and they are the **Memory Blocks** of our LSTM network. They are responsible for choosing what to remember in a sequence and pass on that information to the next cell via two states called the `hidden state` $H_{t}$ and the `cell state` $C_{t}$ where $t$ indicates the time-step. Each `Cell` has dedicated gates which are responsible for storing, writing or reading the information passed to an LSTM. You will now look closely at the architecture of the network by implementing each mechanism happening inside of it. - - -Lets start with writing a function to randomly initialize the parameters which will be learned while our model trains - -```python -def initialise_params(hidden_dim, input_dim): - # forget gate - Wf = rng.standard_normal(size=(hidden_dim, hidden_dim + input_dim)) - bf = rng.standard_normal(size=(hidden_dim, 1)) - # input gate - Wi = rng.standard_normal(size=(hidden_dim, hidden_dim + input_dim)) - bi = rng.standard_normal(size=(hidden_dim, 1)) - # candidate memory gate - Wcm = rng.standard_normal(size=(hidden_dim, hidden_dim + input_dim)) - bcm = rng.standard_normal(size=(hidden_dim, 1)) - # output gate - Wo = rng.standard_normal(size=(hidden_dim, hidden_dim + input_dim)) - bo = rng.standard_normal(size=(hidden_dim, 1)) - - # fully connected layer for classification - W2 = rng.standard_normal(size=(1, hidden_dim)) - b2 = np.zeros((1, 1)) - - parameters = { - "Wf": Wf, - "bf": bf, - "Wi": Wi, - "bi": bi, - "Wcm": Wcm, - "bcm": bcm, - "Wo": Wo, - "bo": bo, - "W2": W2, - "b2": b2 - } - return parameters -``` - -### Forward Propagation - -Now that you have your initialized parameters, you can pass the input data in a forward direction through the network. Each layer accepts the input data, processes it and passes it to the successive layer. This process is called `Forward Propagation`. You will undertake the following mechanism to implement it: -- Loading the word embeddings of the input data -- Passing the embeddings to an LSTM -- Perform all the gate mechanisms in every memory block of the LSTM to obtain the final hidden state -- Passing the final hidden state through a fully connected layer to obtain the probability with which the sequence is positive -- Storing all the calculated values in a cache to utilize during backpropagation - - -[Sigmoid](https://d2l.ai/chapter_multilayer-perceptrons/mlp.html?highlight=sigmoid#sigmoid-function) belongs to the family of non-linear activation functions. It helps the network to update or forget the data. If the sigmoid of a value results in 0, the information is considered forgotten. Similarly, the information stays if it is 1. - -```python -def sigmoid(x): - n = np.exp(np.fmin(x, 0)) - d = (1 + np.exp(-np.abs(x))) - return n / d -``` - -The **Forget Gate** takes the current word embedding and the previous hidden state concatenated together as input. and decides what parts of the old memory cell content need attention and which can be ignored. - -```python -def fp_forget_gate(concat, parameters): - ft = sigmoid(np.dot(parameters['Wf'], concat) - + parameters['bf']) - return ft -``` - -The **Input Gate** takes the current word embedding and the previous hidden state concatenated together as input. and governs how much of the new data we take into account via the **Candidate Memory Gate** which utilizes the [Tanh](https://d2l.ai/chapter_multilayer-perceptrons/mlp.html?highlight=tanh#tanh-function) to regulate the values flowing through the network. - -```python -def fp_input_gate(concat, parameters): - it = sigmoid(np.dot(parameters['Wi'], concat) - + parameters['bi']) - cmt = np.tanh(np.dot(parameters['Wcm'], concat) - + parameters['bcm']) - return it, cmt -``` - -Finally we have the **Output Gate** which takes information from the current word embedding, previous hidden state and the cell state which has been updated with information from the forget and input gates to update the value of the hidden state. - -```python -def fp_output_gate(concat, next_cs, parameters): - ot = sigmoid(np.dot(parameters['Wo'], concat) - + parameters['bo']) - next_hs = ot * np.tanh(next_cs) - return ot, next_hs -``` - -The following image summarizes each gate mechanism in the memory block of a LSTM network: ->Image has been modified from [this](https://link.springer.com/chapter/10.1007%2F978-3-030-14524-8_11) source - -![Diagram showing three sections of a memory block, labeled "Forget gate", "Input gate" and "Output gate". Each gate contains several subparts, representing the operations performed at that stage of the process.](_static/mem_block.png) - -### But how do you obtain sentiment from the LSTM's output? - -The hidden state you obtain from the output gate of the last memory block in a sequence is considered to be a representation of all the information contained in a sequence. To classify this information into various classes (2 in our case, positive and negative) we use a **Fully Connected layer** which firstly maps this information to a predefined output size (1 in our case). Then, an activation function such as the sigmoid converts this output to a value between 0 and 1. We'll consider values greater than 0.5 to be indicative of a positive sentiment. - -```python -def fp_fc_layer(last_hs, parameters): - z2 = (np.dot(parameters['W2'], last_hs) - + parameters['b2']) - a2 = sigmoid(z2) - return a2 -``` - -Now you will put all these functions together to summarize the **Forward Propagation** step in our model architecture: - -```python -def forward_prop(X_vec, parameters, input_dim): - - hidden_dim = parameters['Wf'].shape[0] - time_steps = len(X_vec) - - # Initialise hidden and cell state before passing to first time step - prev_hs = np.zeros((hidden_dim, 1)) - prev_cs = np.zeros(prev_hs.shape) - - # Store all the intermediate and final values here - caches = {'lstm_values': [], 'fc_values': []} - - # Hidden state from the last cell in the LSTM layer is calculated. - for t in range(time_steps): - # Retrieve word corresponding to current time step - x = X_vec[t] - # Retrieve the embedding for the word and reshape it to make the LSTM happy - xt = emb_matrix.get(x, rng.random(size=(input_dim, 1))) - xt = xt.reshape((input_dim, 1)) - - # Input to the gates is concatenated previous hidden state and current word embedding - concat = np.vstack((prev_hs, xt)) - - # Calculate output of the forget gate - ft = fp_forget_gate(concat, parameters) - - # Calculate output of the input gate - it, cmt = fp_input_gate(concat, parameters) - io = it * cmt - - # Update the cell state - next_cs = (ft * prev_cs) + io - - # Calculate output of the output gate - ot, next_hs = fp_output_gate(concat, next_cs, parameters) - - # store all the values used and calculated by - # the LSTM in a cache for backward propagation. - lstm_cache = { - "next_hs": next_hs, - "next_cs": next_cs, - "prev_hs": prev_hs, - "prev_cs": prev_cs, - "ft": ft, - "it" : it, - "cmt": cmt, - "ot": ot, - "xt": xt, - } - caches['lstm_values'].append(lstm_cache) - - # Pass the updated hidden state and cell state to the next time step - prev_hs = next_hs - prev_cs = next_cs - - # Pass the LSTM output through a fully connected layer to - # obtain probability of the sequence being positive - a2 = fp_fc_layer(next_hs, parameters) - - # store all the values used and calculated by the - # fully connected layer in a cache for backward propagation. - fc_cache = { - "a2" : a2, - "W2" : parameters['W2'] - } - caches['fc_values'].append(fc_cache) - return caches -``` - -### Backpropagation - -After each forward pass through the network, you will implement the `backpropagation through time` algorithm to accumulate gradients of each parameter over the time steps. Backpropagation through a LSTM is not as straightforward as through other common Deep Learning architectures, due to the special way its underlying layers interact. Nonetheless, the approach is largely the same; identifying dependencies and applying the chain rule. - - -Lets start with defining a function to initialize gradients of each parameter as arrays made up of zeros with same dimensions as the corresponding parameter - -```python -# Initialise the gradients -def initialize_grads(parameters): - grads = {} - for param in parameters.keys(): - grads[f'd{param}'] = np.zeros((parameters[param].shape)) - return grads -``` - -Now, for each gate and the fully connected layer, we define a function to calculate the gradient of the loss with respect to the input passed and the parameters used. To understand the mathematics behind how the derivatives were calculated we suggest you to follow this helpful [blog](https://christinakouridi.blog/2019/06/19/backpropagation-lstm/) by Christina Kouridi. - - -Define a function to calculate the gradients in the **Forget Gate**: - -```python -def bp_forget_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters): - # dft = dL/da2 * da2/dZ2 * dZ2/dh_prev * dh_prev/dc_prev * dc_prev/dft - dft = ((dc_prev * cache["prev_cs"] + cache["ot"] - * (1 - np.square(np.tanh(cache["next_cs"]))) - * cache["prev_cs"] * dh_prev) * cache["ft"] * (1 - cache["ft"])) - # dWf = dft * dft/dWf - gradients['dWf'] += np.dot(dft, concat.T) - # dbf = dft * dft/dbf - gradients['dbf'] += np.sum(dft, axis=1, keepdims=True) - # dh_f = dft * dft/dh_prev - dh_f = np.dot(parameters["Wf"][:, :hidden_dim].T, dft) - return dh_f, gradients -``` - -Define a function to calculate the gradients in the **Input Gate** and **Candidate Memory Gate**: - -```python -def bp_input_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters): - # dit = dL/da2 * da2/dZ2 * dZ2/dh_prev * dh_prev/dc_prev * dc_prev/dit - dit = ((dc_prev * cache["cmt"] + cache["ot"] - * (1 - np.square(np.tanh(cache["next_cs"]))) - * cache["cmt"] * dh_prev) * cache["it"] * (1 - cache["it"])) - # dcmt = dL/da2 * da2/dZ2 * dZ2/dh_prev * dh_prev/dc_prev * dc_prev/dcmt - dcmt = ((dc_prev * cache["it"] + cache["ot"] - * (1 - np.square(np.tanh(cache["next_cs"]))) - * cache["it"] * dh_prev) * (1 - np.square(cache["cmt"]))) - # dWi = dit * dit/dWi - gradients['dWi'] += np.dot(dit, concat.T) - # dWcm = dcmt * dcmt/dWcm - gradients['dWcm'] += np.dot(dcmt, concat.T) - # dbi = dit * dit/dbi - gradients['dbi'] += np.sum(dit, axis=1, keepdims=True) - # dWcm = dcmt * dcmt/dbcm - gradients['dbcm'] += np.sum(dcmt, axis=1, keepdims=True) - # dhi = dit * dit/dh_prev - dh_i = np.dot(parameters["Wi"][:, :hidden_dim].T, dit) - # dhcm = dcmt * dcmt/dh_prev - dh_cm = np.dot(parameters["Wcm"][:, :hidden_dim].T, dcmt) - return dh_i, dh_cm, gradients -``` - -Define a function to calculate the gradients for the **Output Gate**: - -```python -def bp_output_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters): - # dot = dL/da2 * da2/dZ2 * dZ2/dh_prev * dh_prev/dot - dot = (dh_prev * np.tanh(cache["next_cs"]) - * cache["ot"] * (1 - cache["ot"])) - # dWo = dot * dot/dWo - gradients['dWo'] += np.dot(dot, concat.T) - # dbo = dot * dot/dbo - gradients['dbo'] += np.sum(dot, axis=1, keepdims=True) - # dho = dot * dot/dho - dh_o = np.dot(parameters["Wo"][:, :hidden_dim].T, dot) - return dh_o, gradients -``` - -Define a function to calculate the gradients for the **Fully Connected Layer**: - -```python -def bp_fc_layer (target, caches, gradients): - # dZ2 = dL/da2 * da2/dZ2 - predicted = np.array(caches['fc_values'][0]['a2']) - target = np.array(target) - dZ2 = predicted - target - # dW2 = dL/da2 * da2/dZ2 * dZ2/dW2 - last_hs = caches['lstm_values'][-1]["next_hs"] - gradients['dW2'] = np.dot(dZ2, last_hs.T) - # db2 = dL/da2 * da2/dZ2 * dZ2/db2 - gradients['db2'] = np.sum(dZ2) - # dh_last = dZ2 * W2 - W2 = caches['fc_values'][0]["W2"] - dh_last = np.dot(W2.T, dZ2) - return dh_last, gradients -``` - -Put all these functions together to summarize the **Backpropagation** step for our model: - -```python -def backprop(y, caches, hidden_dim, input_dim, time_steps, parameters): - - # Initialize gradients - gradients = initialize_grads(parameters) - - # Calculate gradients for the fully connected layer - dh_last, gradients = bp_fc_layer(target, caches, gradients) - - # Initialize gradients w.r.t previous hidden state and previous cell state - dh_prev = dh_last - dc_prev = np.zeros((dh_prev.shape)) - - # loop back over the whole sequence - for t in reversed(range(time_steps)): - cache = caches['lstm_values'][t] - - # Input to the gates is concatenated previous hidden state and current word embedding - concat = np.concatenate((cache["prev_hs"], cache["xt"]), axis=0) - - # Compute gates related derivatives - # Calculate derivative w.r.t the input and parameters of forget gate - dh_f, gradients = bp_forget_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters) - - # Calculate derivative w.r.t the input and parameters of input gate - dh_i, dh_cm, gradients = bp_input_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters) - - # Calculate derivative w.r.t the input and parameters of output gate - dh_o, gradients = bp_output_gate(hidden_dim, concat, dh_prev, dc_prev, cache, gradients, parameters) - - # Compute derivatives w.r.t prev. hidden state and the prev. cell state - dh_prev = dh_f + dh_i + dh_cm + dh_o - dc_prev = (dc_prev * cache["ft"] + cache["ot"] - * (1 - np.square(np.tanh(cache["next_cs"]))) - * cache["ft"] * dh_prev) - - return gradients -``` - -### Updating the Parameters - -We update the parameters through an optimization algorithm called [Adam](https://optimization.cbe.cornell.edu/index.php?title=Adam) which is an extension to stochastic gradient descent that has recently seen broader adoption for deep learning applications in computer vision and natural language processing. Specifically, the algorithm calculates an exponential moving average of the gradient and the squared gradient, and the parameters `beta1` and `beta2` control the decay rates of these moving averages. Adam has shown increased convergence and robustness over other gradient descent algorithms and is often recommended as the default optimizer for training. - - -Define a function to initialise the moving averages for each parameter - -```python -# initialise the moving averages -def initialise_mav(hidden_dim, input_dim, params): - v = {} - s = {} - # Initialize dictionaries v, s - for key in params: - v['d' + key] = np.zeros(params[key].shape) - s['d' + key] = np.zeros(params[key].shape) - # Return initialised moving averages - return v, s -``` - -Define a function to update the parameters - -```python -# Update the parameters using Adam optimization -def update_parameters(parameters, gradients, v, s, - learning_rate=0.01, beta1=0.9, beta2=0.999): - for key in parameters: - # Moving average of the gradients - v['d' + key] = (beta1 * v['d' + key] - + (1 - beta1) * gradients['d' + key]) - - # Moving average of the squared gradients - s['d' + key] = (beta2 * s['d' + key] - + (1 - beta2) * (gradients['d' + key] ** 2)) - - # Update parameters - parameters[key] = (parameters[key] - learning_rate - * v['d' + key] / np.sqrt(s['d' + key] + 1e-8)) - # Return updated parameters and moving averages - return parameters, v, s -``` - -### Training the Network - - -You will start by initializing all the parameters and hyperparameters being used in your network - -```python -hidden_dim = 64 -input_dim = emb_matrix['memory'].shape[0] -learning_rate = 0.001 -epochs = 10 -parameters = initialise_params(hidden_dim, - input_dim) -v, s = initialise_mav(hidden_dim, - input_dim, - parameters) -``` - -To optimize your deep learning network, you need to calculate a loss based on how well the model is doing on the training data. Loss value implies how poorly or well a model behaves after each iteration of optimization.
-Define a function to calculate the loss using [negative log likelihood](http://d2l.ai/chapter_linear-networks/softmax-regression.html?highlight=negative%20log%20likelihood#log-likelihood) - -```python -def loss_f(A, Y): - # define value of epsilon to prevent zero division error inside a log - epsilon = 1e-5 - # Implement formula for negative log likelihood - loss = (- Y * np.log(A + epsilon) - - (1 - Y) * np.log(1 - A + epsilon)) - # Return loss - return np.squeeze(loss) -``` - -Set up the neural network's learning experiment with a training loop and start the training process. You will also evaluate the model's performance on the training dataset to see how well the model is *learning* and the testing dataset to see how well it is *generalizing*. ->Skip running this cell if you already have the trained parameters stored in a `npy` file - -```python -# To store training losses -training_losses = [] -# To store testing losses -testing_losses = [] - -# This is a training loop. -# Run the learning experiment for a defined number of epochs (iterations). -for epoch in range(epochs): - ################# - # Training step # - ################# - train_j = [] - for sample, target in zip(X_train, y_train): - # split text sample into words/tokens - b = textproc.word_tokeniser(sample) - - # Forward propagation/forward pass: - caches = forward_prop(b, - parameters, - input_dim) - - # Backward propagation/backward pass: - gradients = backprop(target, - caches, - hidden_dim, - input_dim, - len(b), - parameters) - - # Update the weights and biases for the LSTM and fully connected layer - parameters, v, s = update_parameters(parameters, - gradients, - v, - s, - learning_rate=learning_rate, - beta1=0.999, - beta2=0.9) - - # Measure the training error (loss function) between the actual - # sentiment (the truth) and the prediction by the model. - y_pred = caches['fc_values'][0]['a2'][0][0] - loss = loss_f(y_pred, target) - # Store training set losses - train_j.append(loss) - - ################### - # Evaluation step # - ################### - test_j = [] - for sample, target in zip(X_test, y_test): - # split text sample into words/tokens - b = textproc.word_tokeniser(sample) - - # Forward propagation/forward pass: - caches = forward_prop(b, - parameters, - input_dim) - - # Measure the testing error (loss function) between the actual - # sentiment (the truth) and the prediction by the model. - y_pred = caches['fc_values'][0]['a2'][0][0] - loss = loss_f(y_pred, target) - - # Store testing set losses - test_j.append(loss) - - # Calculate average of training and testing losses for one epoch - mean_train_cost = np.mean(train_j) - mean_test_cost = np.mean(test_j) - training_losses.append(mean_train_cost) - testing_losses.append(mean_test_cost) - print('Epoch {} finished. \t Training Loss : {} \t Testing Loss : {}'. - format(epoch + 1, mean_train_cost, mean_test_cost)) - -# save the trained parameters to a npy file -np.save('tutorial-nlp-from-scratch/parameters.npy', parameters) -``` - -It is a good practice to plot the training and testing losses as the learning curves are often helpful in diagnosing the behavior of a Machine Learning model. - -```python -fig = plt.figure() -ax = fig.add_subplot(111) - -# plot the training loss -ax.plot(range(0, len(training_losses)), training_losses, label='training loss') -# plot the testing loss -ax.plot(range(0, len(testing_losses)), testing_losses, label='testing loss') - -# set the x and y labels -ax.set_xlabel("epochs") -ax.set_ylabel("loss") - -plt.legend(title='labels', bbox_to_anchor=(1.0, 1), loc='upper left') -plt.show() -``` - -### Sentiment Analysis on the Speech Data - - -Once your model is trained, you can use the updated parameters to start making our predictions. You can break each speech into paragraphs of uniform size before passing them to the Deep Learning model and predicting the sentiment of each paragraph - -```python -# To store predicted sentiments -predictions = {} - -# define the length of a paragraph -para_len = 100 - -# Retrieve trained values of the parameters -if os.path.isfile('tutorial-nlp-from-scratch/parameters.npy'): - parameters = np.load('tutorial-nlp-from-scratch/parameters.npy', allow_pickle=True).item() - -# This is the prediction loop. -for index, text in enumerate(X_pred): - # split each speech into paragraphs - paras = textproc.text_to_paras(text, para_len) - # To store the network outputs - preds = [] - - for para in paras: - # split text sample into words/tokens - para_tokens = textproc.word_tokeniser(para) - # Forward Propagation - caches = forward_prop(para_tokens, - parameters, - input_dim) - - # Retrieve the output of the fully connected layer - sent_prob = caches['fc_values'][0]['a2'][0][0] - preds.append(sent_prob) - - threshold = 0.5 - preds = np.array(preds) - # Mark all predictions > threshold as positive and < threshold as negative - pos_indices = np.where(preds > threshold) # indices where output > 0.5 - neg_indices = np.where(preds < threshold) # indices where output < 0.5 - # Store predictions and corresponding piece of text - predictions[speakers[index]] = {'pos_paras': paras[pos_indices[0]], - 'neg_paras': paras[neg_indices[0]]} -``` - -Visualizing the sentiment predictions: - -```python -x_axis = [] -data = {'positive sentiment': [], 'negative sentiment': []} -for speaker in predictions: - # The speakers will be used to label the x-axis in our plot - x_axis.append(speaker) - # number of paras with positive sentiment - no_pos_paras = len(predictions[speaker]['pos_paras']) - # number of paras with negative sentiment - no_neg_paras = len(predictions[speaker]['neg_paras']) - # Obtain percentage of paragraphs with positive predicted sentiment - pos_perc = no_pos_paras / (no_pos_paras + no_neg_paras) - # Store positive and negative percentages - data['positive sentiment'].append(pos_perc*100) - data['negative sentiment'].append(100*(1-pos_perc)) - -index = pd.Index(x_axis, name='speaker') -df = pd.DataFrame(data, index=index) -ax = df.plot(kind='bar', stacked=True) -ax.set_ylabel('percentage') -ax.legend(title='labels', bbox_to_anchor=(1, 1), loc='upper left') -plt.show() -``` - -In the plot above, you're shown what percentages of each speech are expected to carry a positive and negative sentiment. Since this implementation prioritized simplicity and clarity over performance, we cannot expect these results to be very accurate. Moreover, while making the sentiment predictions for one paragraph we did not use the neighboring paragraphs for context which would have led to more accurate predictions. We encourage the reader to play around with the model and make some tweaks suggested in `Next Steps` and observe how the model performance changes. - - -## Looking at our Neural Network from an ethical perspective - - -It's crucial to understand that accurately identifying a text's sentiment is not easy primarily because of the complex ways in which humans express sentiment, using irony, sarcasm, humor, or, in social media, abbreviation. Moreover neatly placing text into two categories: 'positive' and 'negative' can be problematic because it is being done without any context. Words or abbreviations can convey very different sentiments depending on age and location, none of which we took into account while building our model. - -Along with data, there are also growing concerns that data processing algorithms are influencing policy and daily lives in ways that are not transparent and introduce biases. Certain biases such as the [Inductive Bias](https://bit.ly/2WtTKIe) are essential to help a Machine Learning model generalize better, for example the LSTM we built earlier is biased towards preserving contextual information over long sequences which makes it so suitable for processing sequential data. The problem arises when [societal biases](https://hbr.org/2019/10/what-do-we-do-about-the-biases-in-ai) creep into algorithmic predictions. Optimizing Machine algorithms via methods like [hyperparameter tuning](https://en.wikipedia.org/wiki/Hyperparameter_optimization) can then further amplify these biases by learning every bit of information in the data. - - -There are also cases where bias is only in the output and not the inputs (data, algorithm). For example, in sentiment analysis [accuracy tends to be higher on female-authored texts than on male-authored ones]( https://doi.org/10.3390/electronics9020374). End users of sentiment analysis should be aware that its small gender biases can affect the conclusions drawn from it and apply correction factors when necessary. Hence, it is important that demands for algorithmic accountability should include the ability to test the outputs of a system, including the ability to drill down into different user groups by gender, ethnicity and other characteristics, to identify, and hopefully suggest corrections for, system output biases. - - -## Next Steps - - -You have learned how to build and train a simple Long Short Term Memory network from scratch using just NumPy to perform sentiment analysis. - -To further enhance and optimize your neural network model, you can consider one of a mixture of the following: - -- Alter the architecture by introducing multiple LSTM layers to make the network deeper. -- Use a higher epoch size to train longer and add more regularization techniques, such as early stopping, to prevent overfitting. -- Introduce a validation set for an unbiased evaluation of the model fit. -- Apply batch normalization for faster and more stable training. -- Tune other parameters, such as the learning rate and hidden layer size. -- Initialize weights using [Xavier Initialization](https://d2l.ai/chapter_multilayer-perceptrons/numerical-stability-and-init.html#xavier-initialization) to prevent vanishing/exploding gradients instead of initializing them randomly. -- Replace LSTM with a [Bidirectional LSTM](https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks) to use both left and right context for predicting sentiment. - -Nowadays, LSTMs have been replaced by the [Transformer](https://jalammar.github.io/illustrated-transformer/)( which uses [Attention](https://jalammar.github.io/visualizing-neural-machine-translation-mechanics-of-seq2seq-models-with-attention/) to tackle all the problems that plague an LSTM such as as lack of [transfer learning](https://en.wikipedia.org/wiki/Transfer_learning), lack of [parallel training](https://web.stanford.edu/~rezab/classes/cme323/S16/projects_reports/hedge_usmani.pdf) and a long gradient chain for lengthy sequences - -Building a neural network from scratch with NumPy is a great way to learn more about NumPy and about deep learning. However, for real-world applications you should use specialized frameworks — such as PyTorch, JAX, TensorFlow or MXNet — that provide NumPy-like APIs, have built-in automatic differentiation and GPU support, and are designed for high-performance numerical computing and machine learning. - -Finally, to know more about how ethics come into play when developing a machine learning model, you can refer to the following resources : -- Data ethics resources by the Turing Institute. https://www.turing.ac.uk/research/data-ethics -- Considering how artificial intelligence shifts power, an [article](https://www.nature.com/articles/d41586-020-02003-2) and [talk](https://slideslive.com/38923453/the-values-of-machine-learning) by Pratyusha Kalluri -- More ethics resources on [this blog post](https://www.fast.ai/2018/09/24/ai-ethics-resources/) by Rachel Thomas and the [Radical AI podcast](https://www.radicalai.org/) diff --git a/content/tutorial-nlp-from-scratch/README.md b/content/tutorial-nlp-from-scratch/README.md deleted file mode 100644 index 67bffe7b..00000000 --- a/content/tutorial-nlp-from-scratch/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Data used for building the [NLP from scratch tutorial](https://github.com/Dbhasin1/numpy-tutorials/blob/ethics-tutorial/content/tutorial-nlp-from-scratch.md) - -## [IMDb Reviews Dataset](https://github.com/Dbhasin1/numpy-tutorials/blob/ethics-tutorial/content/tutorial-nlp-from-scratch/IMDB%20Dataset.csv) - -**Purpose**: Training the Deep Learning model - -> Information courtesy of -IMDb -(http://www.imdb.com). -Used with permission. - -IMDB Reviews Dataset is a large movie review dataset collected and prepared by -Andrew L. Maas from the popular movie rating service, IMDB. The IMDB Reviews -dataset is used for binary sentiment classification, whether a review is -positive or negative. It contains 25,000 movie reviews for training and 25,000 -for testing. All these 50,000 reviews are labeled data that may be used for -supervised deep learning. For ease of reproducibility, we'll be sourcing the -data from [Zenodo](https://zenodo.org/record/4117827#.YVQZ_EZBy3Ihttps://zenodo.org/record/4117827#.YVQZ_EZBy3I). - -> Andrea Esuli, Alejandro Moreo, & Fabrizio Sebastiani. (2020). Sentiment -Quantification Datasets [Data set]. Zenodo. -https://doi.org/10.5281/zenodo.4117827 - ---- - -## [Glove Embeddings](https://github.com/Dbhasin1/numpy-tutorials/blob/ethics-tutorial/content/tutorial-nlp-from-scratch/glove.6B.50d.txt) - -**Purpose**: To represent text data in machine-readable i.e numeric format -> Jeffrey Pennington, Richard Socher, and Christopher D. Manning. 2014. -[GloVe: Global Vectors for Word Representation](https://nlp.stanford.edu/pubs/glove.pdf) - -GloVe is an unsupervised algorithm developed for generating word embeddings by -generating global word-word co-occurence matrix from a corpus. You can download -the zipped files containing the embeddings from -https://nlp.stanford.edu/projects/glove/. -Here you can choose any of the four options for different sizes or training -datasets, we opted for the least resource-heavy file with 50 dimensional -representations for each word. - ---- - -## [Speech Dataset](https://github.com/Dbhasin1/numpy-tutorials/blob/ethics-tutorial/content/tutorial-nlp-from-scratch/speeches.csv) - -**Purpose**: The trained Deep Learning Model will perform sentiment analysis on -this data -> Curated by the authors of the tutorial - -We have chosen speeches by activists around the globe talking about issues like -climate change, feminism, lgbtqa+ rights and racism. These were sourced from -newspapers, the official website of the United Nations and the archives of -established universities as cited in the table below. A CSV file was created -containing the transcribed speeches, their speaker and the source the speeches -were obtained from. -We made sure to include different demographics in our data and included a range -of different topics, most of which focus on social and/or ethical issues. The -dataset is subjected to the CC0 Creative Common License, which means that is -free for the public to use and there are no copyrights reserved. - -| Speech | Speaker | Source | -|--------------------------------------------------|-------------------------|------------------------------------------------------------| -| Barnard College Commencement | Leymah Gbowee | [Barnard College](https://barnard.edu/news/transcript-speech-nobel-peace-prize-winner-leymah-gbowee) | -| UN Speech on youth Education | Malala Yousafzai | [The Guardian](https://www.theguardian.com/commentisfree/2013/jul/12/malala-yousafzai-united-nations-education-speech-text) | -| Remarks in the UNGA on racial discrimination | Linda Thomas Greenfield | [United States mission to the United Nation](https://usun.usmission.gov/remarks-by-ambassador-linda-thomas-greenfield-at-a-un-general-assembly-commemorative-meeting-for-intl-day-for-the-elimination-of-racial-discrimination/) | -| How Dare You | Greta Thunberg | [NBC](https://www.nbcnews.com/news/world/read-greta-thunberg-s-full-speech-united-nations-climate-action-n1057861) | -| The speech that silenced the world for 5 minutes | Severn Suzuki | [Earth Charter](https://earthcharter.org/new-voices-after-26-years-of-the-girl-who-silenced-the-world-for-5-minutes/) | -| The Hope Speech | Harvey Milk | [Museum of Fine Arts, Boston](https://www.mfa.org/exhibitions/amalia-pica/transcript-harvey-milks-the-hope-speech) | -| Speech at the time to Thrive Conference | Ellen Page | [Huffpost](https://www.huffpost.com/entry/time-to-thrive_b_4794251) | -| I have a dream | Martin Luther King | [Marshall University](https://www.marshall.edu/onemarshallu/i-have-a-dream/) | diff --git a/content/tutorial-nlp-from-scratch/dl_architectures.jpg b/content/tutorial-nlp-from-scratch/dl_architectures.jpg deleted file mode 100644 index f9c07677..00000000 Binary files a/content/tutorial-nlp-from-scratch/dl_architectures.jpg and /dev/null differ diff --git a/content/tutorial-nlp-from-scratch/speeches.csv b/content/tutorial-nlp-from-scratch/speeches.csv deleted file mode 100644 index 3e4734e2..00000000 --- a/content/tutorial-nlp-from-scratch/speeches.csv +++ /dev/null @@ -1,113 +0,0 @@ -speaker,speech,source -Greta Thunberg,"""My message is that we'll be watching you. This is all wrong. I shouldn't be up here. I should be back in school on the other side of the ocean. Yet you all come to us young people for hope. How dare you! ""You have stolen my dreams and my childhood with your empty words. And yet I'm one of the lucky ones. People are suÔ¨Äering. People are dying. Entire ecosystems are collapsing. We are in the beginning of a mass extinction, and all you can talk about is money and fairy tales of eternal economic growth. How dare you! ""For more than 30 years, the science has been crystal clear. How dare you continue to look away and come here saying that you're doing enough, when the politics and solutions needed are still nowhere in sight. ""You say you hear us and that you understand the urgency. But no matter how sad and angry I am, I do not want to believe that. Because if you really understood the situation and still kept on failing to act, then you would be evil. And that I refuse to believe. ""The popular idea of cutting our emissions in half in 10 years only gives us a 50% chance of staying below 1.5¬∞C, and the risk of setting off irreversible chain reactions beyond human control. ""Fifty percent may be acceptable to you. But those numbers do not include tipping points, most feedback loops, additional warming hidden by toxic air pollution or the aspects of equity and climate justice. They also rely on my generation sucking hundreds of billions of tons of your CO2 out of the air with technologies that barely exist. ""So a 50% risk is simply not acceptable to us ‚Äî we who have to live with the consequences. ""To have a 67% chance of staying below a 1.5¬∞C global temperature rise ‚Äì the best odds given by the [Intergovernmental Panel on Climate Change] ‚Äì the world had 420 gigatons of CO2 left to emit back on January 1st, 2018. Today that figure is already down to less than 350 gigatons. ""How dare you pretend that this can be solved with just 'business as usual' and some technical solutions? With today's emissions levels, that remaining CO2 budget will be entirely gone within less than 8 1/2 years. ""There will not be any solutions or plans presented in line with these figures here today, because these numbers are too uncomfortable. And you are still not mature enough to tell it like it is. ""You are failing us. But young people are starting to understand your betrayal. The eyes of all future generations are upon you. And if you choose to fail us, I say: We will never forgive you. ""We will not let you get away with this. Right here, right now is where we draw the line. The world is waking up. And change is coming, whether you like it or not. ""Thank you.""",NBC official website -Severn Suzuki,"Hello, I'm Severn Suzuki speaking for ""ECO"" -- the Environmental Children‚Äôs Organization. We are a group of 12 and 13 year-olds trying to make a difference: Vanessa Suttie, Morgan Geisler, Michelle Quigg, and me. We‚Äôve raised all the money to come here ourselves -- to come 5,000 miles to tell you adults you must change your ways. Coming up here today, I have no hidden agenda. I am fighting for my future. Losing my future is not like losing an election, or a few points on the stock market. I am here to speak for all generations to come. I am here to speak -- speak on behalf of the starving children around the world whose cries go unheard. I am here to speak for the countless animals dying across this planet, because they have nowhere left to go. I am afraid to go out in the sun now, because of the holes in our ozone. I am afraid to breathe the air, because I don‚Äôt know what chemicals are in it. I used to go in -- I used to go fishing in Vancouver, my home, with my Dad until, just a few years ago, we found the fish full of cancers. And now we hear of animals and plants going extinct every day, vanishing forever. In my life, I have dreamt of seeing the great herds of wild animals, jungles, and rainforests full of birds and butterflies, but now I wonder if they will even exist for my children to see. Did you have to worry of these things when you were my age? All this is happening before our eyes and yet we act as if we have all the time we want and all the solutions. I‚Äôm only a child and I don‚Äôt have all the solutions. I don't -- I want you to realize, neither do you. You don‚Äôt know how to fix the holes in our ozone layer. You don‚Äôt know how to bring the salmon back up in a dead stream. You don‚Äôt know how to bring back an animal now extinct. And you can‚Äôt bring back the forests that once grew where there is now a desert. If you don‚Äôt know how to fix it, please stop breaking it. Here, you may be delegates of your governments, business people, organizers, reporters, or politicians. But, really, you are mothers and fathers, sisters and brothers, aunts and uncles -- and all of you are someone‚Äôs child. I‚Äôm only a child, yet I know we are all part of a family -- five billion strong; in fact 30 million species strong -and borders and governments will never change that. I‚Äôm only a child, yet I know we are all in this together and should act as one single world towards one single goal. But, really, you are mothers and fathers, sisters and brothers, aunts and uncles -- and all of you are someone‚Äôs child. I‚Äôm only a child, yet I know we are all part of a family -- five billion strong; in fact 30 million species strong -and borders and governments will never change that. I‚Äôm only a child, yet I know we are all in this together and should act as one single world towards one single goal. In -- In my anger, I'm not blind; and in my fear, I'm not afraid of telling the world how I feel. In my country we make so much waste, we buy and throw away, buy and throw away, buy and throw away and yet Northern countries will not share with the needy. Even when we have more than enough we are afraid to share; we are afraid to let go of some of our wealth. In Canada, we live the privileged life. We‚Äôve plenty of food, water, and shelter. We have watches, bicycles, computers, and television sets. The list could go on for two days. Two days ago, here in Brazil, we were shocked when we spent time with some children living on the streets. This is what one child told us: ""I wish I was rich and if I were, I would give all the street children food, clothes, medicines, shelter, and love and affection."" If a child on the streets who has nothing is willing to share, why are we who have everything still so greedy? I can‚Äôt stop thinking that these are children my own age, that it makes a tremendous difference where you are born; that I could be one of those children living in the favelas of Rio. I could be a child starving in Somalia, or a victim of war in the Middle East, or a beggar in India. I am only a child, yet I know if all the money spent on war was spent on finding environmental answers ending poverty and in finding treaties, what a wonderful place this Earth would be. At school, even in kindergarten, you teach us how to behave in the world. You teach us to not to fight with others, to work things out, to respect others, to clean up our mess, not to hurt other creatures, to share, not be greedy. Then, why do you go out and do -- do the things you tell us not to do? Do not forget why you are attending these conferences -- who you're doing this At school, even in kindergarten, you teach us how to behave in the world. You teach us to not to fight with others, to work things out, to respect others, to clean up our mess, not to hurt other creatures, to share, not be greedy. Then, why do you go out and do -- do the things you tell us not to do? Do not forget why you are attending these conferences -- who you're doing this for. We are your own children. You are deciding what kind of a world we are growing up in. Parents should be able to comfort their children by saying, ""Everything's going to be all right; it‚Äôs not the end of the world, and we're -- and we're doing the best we can."" But I don‚Äôt think you can say that to us anymore. Are we even on your list of priorities? My dad always says, ""You are what you do, not what you say."" Well, what you do makes me cry at night. You grown-ups say you love us. But I challenge you, please, make your actions reflect your words. Thank you.",NTU blogs -Martin Luther King,"I am happy to join with you today in what will go down in history as the greatest demonstration for freedom in the history of our nation. Five score years ago a great American in whose symbolic shadow we stand today signed the Emancipation Proclamation. This momentous decree is a great beacon light of hope to millions of Negro slaves who had been seared in the flames of withering injustice. It came as a joyous daybreak to end the long night of their captivity. But 100 years later the Negro still is not free. One hundred years later the life of the Negro is still badly crippled by the manacles of segregation and the chains of discrimination. One hundred years later the Negro lives on a lonely island of poverty in the midst of a vast ocean of material prosperity. One hundred years later the Negro is still languished in the corners of American society and finds himself in exile in his own land. So we‚Äôve come here today to dramatize a shameful condition. In a sense we‚Äôve come to our nation‚Äôs capital to cash a check. When the architects of our Republic wrote the magnificent words of the Constitution and the Declaration of Independence, they were signing a promissory note to which every American was to fall heir. This note was a promise that all men‚Äîyes, black men as well as white men‚Äîwould be guaranteed the unalienable rights of life, liberty and the pursuit of happiness. . . . We must forever conduct our struggle on the high plane of dignity and discipline. We must not allow our creative protests to degenerate into physical violence. . . . The marvelous new militancy which has engulfed the Negro community must not lead us to distrust all white people, for many of our white brothers, as evidenced by their presence here today, have come to realize that their destiny is tied up with our destiny. . . . We cannot walk alone. And as we walk we must make the pledge that we shall always march ahead. We cannot turn back. There are those who are asking the devotees of civil rights, ‚ÄúWhen will you be satisfied?‚Äù We can never be satisfied as long as the Negro is the victim of the unspeakable horrors of police brutality. We can never be satisfied as long as our bodies, heavy with the fatigue of travel, cannot gain lodging in the motels of the highways and the hotels of the cities. We cannot be satisfied as long as the Negro‚Äôs basic mobility is from a smaller ghetto to a larger one. We can never be satisfied as long as our children are stripped of their adulthood and robbed of their dignity by signs stating ‚ÄúFor Whites Only.‚Äù We cannot be satisfied as long as the Negro in Mississippi cannot vote and the Negro in New York believes he has nothing for which to vote. No, no, we are not satisfied, and we will not be satisfied until justice rolls down like waters and righteousness like a mighty stream. . . . I say to you today, my friends, though, even though we face the difficulties of today and tomorrow, I still ¬©2014 The Gilder Lehrman Institute of American History www.gilderlehrman.org have a dream. It is a dream deeply rooted in the American dream. I have a dream that one day this nation will rise up, live out the true meaning of its creed: ‚ÄúWe hold these truths to be self-evident, that all men are created equal.‚Äù I have a dream that one day on the red hills of Georgia sons of former slaves and the sons of former slave-owners will be able to sit down together at the table of brotherhood. I have a dream that one day even the state of Mississippi, a state sweltering with the heat of injustice, sweltering with the heat of oppression, will be transformed into an oasis of freedom and justice. I have a dream that my four little children will one day live in a nation where they will not be judged by the color of their skin but by the content of their character. I have a dream . . . I have a dream that one day in Alabama, with its vicious racists, with its governor having his lips dripping with the words of interposition and nullification, one day right there in Alabama little black boys and black girls will be able to join hands with little white boys and white girls as sisters and brothers. I have a dream today . . . This will be the day when all of God‚Äôs children will be able to sing with new meaning. ‚ÄúMy country, ‚Äôtis of thee, sweet land of liberty, of thee I sing. Land where my fathers died, land of the pilgrim‚Äôs pride, from every mountain side, let freedom ring.‚Äù And if America is to be a great nation, this must become true. So let freedom ring from the prodigious hilltops of New Hampshire. Let freedom ring from the mighty mountains of New York. Let freedom ring from the heightening Alleghenies of Pennsylvania. Let freedom ring from the snowcapped Rockies of Colorado. Let freedom ring from the curvaceous slopes of California. But not only that. Let freedom ring from Stone Mountain of Georgia. Let freedom ring from Lookout Mountain of Tennessee. Let freedom ring from every hill and molehill of Mississippi, from every mountain side. Let freedom ring . . . When we allow freedom to ring‚Äîwhen we let it ring from every city and every hamlet, from every state and every city, we will be able to speed up that day when all of God‚Äôs children, black men and white men, Jews and Gentiles, Protestants and Catholics, will be able to join hands and sing in the words of the old Negro spiritual, ‚ÄúFree at last, Free at last, Great God amighty, We are free at last.‚Äù Reprinted by arrangement with The Heirs to the Estate of Martin Luther King Jr., c/ o Writers House as the proprietor New York, NY. Copyright: ¬© 1963 Dr. Martin Luther King Jr. ¬© renewed 1991 Coretta Scott King.",Brittanica official website -Harvey Milk,"About six months ago, Anita Bryant in her speaking to God said that the drought in California was because of the gay people. On November 9, the day after I got elected, it started to rain. On the day I got sworn in, we walked to City Hall and it was kind of nice, and as soon as I said the word ""I do,"" it started to rain again. It's been raining since then and the people of San Francisco figure the only way to stop it is to do a recall petition. So much for that. Why are we here? Why are gay people here? And what's happening? Let's look at 1977. In 1977, gay people had their rights taken away from them in Miami. But you must remember that in the week before Miami and the week after that, the word homosexual or gay appeared in every single newspaper in this nation in articles both pro and con. In every radio station, in every TV station and every household. For the first time in the history of the world, everybody was talking about it, good or bad. Unless you have dialogue, unless you open the walls of dialogue, you can never reach to change people's opinion. Once you have dialogue starting, you know you can break down prejudice. In 1977 we saw a dialogue start. In 1977, we saw a gay person elected in San Francisco. What that is, is a record of what happened last year. What we must do is make sure that 1978 continues the movement. I know we are pressed for time so I'm going to cover just one more little point. That is to understand why it is important that gay people run for office and that gay people get elected. I know there are many people in this room who are running for central committee who are gay. I encourage you. There's a major reason why. If my non-gay friends and supporters in this room understand it, they'll probably understand why I've run so often before I finally made it. You see there is a major difference ‚Äì and it remains a vital difference ‚Äì between a friend and a gay person, a friend in office and a gay person in office. Gay people have been slandered nationwide. We've been tarred and we've been brushed with the picture of pornography. In Dade County, we were accused of child molestation. It's not enough anymore just to have friends represent us. No matter how good that friend may be. The black community made up its mind to that a long time ago. That the myths against blacks can only be dispelled by electing black leaders, so the black community could be judged by the leaders and not by the myths or black criminals. The Spanish community must not be judged by Latin criminals or myths. The Asian community must not be judged by Asian criminals or myths. The Italian community must not be judged by the mafia, myths. And the time has come when the gay community must not be judged by our criminals and myths. Like every other group, we must be judged by our leaders and by those who are themselves gay, those who are visible. For invisible, we remain in limbo - a myth, a person with no parents, no brothers, no sisters, no friends who are straight, no important positions in employment. A tenth of the nation supposedly composed of stereotypes and would-be seducers of children ‚Äì and no offense meant to the stereotypes. But today, the black community is not judged by its friends, but by its black legislators and leaders. And we must give people the chance to judge us by our leaders and legislators. A gay person in office can set a tone, can command respect not only from the larger community, but from the young people in our own community who need both examples and hope. The first gay people we elect must be strong. They must not be content to sit in the back of the bus. They must not be content to accept pabulum. They must be above wheeling and dealing. They must be ‚Äì for the good of all of us ‚Äì independent, unbought. The anger and the frustrations that some of us feel is because we are misunderstood, and friends can't feel the anger and frustration. They can sense it in us, but they can't feel it. Because a friend has never gone through what is known as coming out. I will never forget what it was like coming out and having nobody to look up toward. I remember the lack of hope - and our friends can't fulfil it. I can't forget the looks on faces of people who've lost hope. Be they gay, be they seniors, be they blacks looking for an almostimpossible job, be they Latins trying to explain their problems and aspirations in a tongue that's foreign to them. I personally will never forget that people are more important than buildings. I use the word ""I"" because I'm proud. I stand here tonight in front of my gay sisters, brothers and friends because I'm proud of you. I think it's time that we have many legislators who are gay and proud of that fact and do not have to remain in the closet. I think that a gay person, up-front, will not walk away from a responsibility and be afraid of being tossed out of office. After Dade County, I walked among the angry and the frustrated night after night and I looked at their faces. And in San Francisco, three days before Gay Pride Day, a person was killed just because he was gay. And that night, I walked among the sad and the frustrated at City Hall in San Francisco and later that night as they lit candles on Castro Street and stood in silence, reaching out for some symbolic thing that would give them hope. These were strong people, whose faces I knew from the shop, the streets, meetings and people who I never saw before but I knew. They were strong, but even they needed hope. And the young gay people who are coming out and hear Anita Bryant on television and her story. The only thing they have to look forward to is hope. And you have to give them hope. Hope for a better world, hope for a better tomorrow, hope for a better place to come to if the pressures at home are too great. Hope that all will be all right. Without hope, not only gays, but the blacks, the seniors, the handicapped, the us'es, the us'es will give up. And if you help elect to the central committee and other offices, more gay people, that gives a green light to all who feel disenfranchised, a green light to move forward. It means hope to a nation that has given up, because if a gay person makes it, the doors are open to everyone. So if there is a message I have to give, it is that I've found one overriding thing about my personal election, it's the fact that if a gay person can be elected, it's a green light. And you and you and you, you have to give people hope.",University of Maryland archives -Leymah Gbowee,"Thank you. Please have your seats. Someone once told me, the kids in America are born with whistles in their bellies. There is nowhere in the world that girls can scream like America. Thank you, President Spar. This is truly an honor. President Spar, Provost Bell, Board Chair Caruso, Dean Hinkson, faculty, student body, special guests, proud parents, distinguished ladies and gentleman, I‚Äôm honored to be here today at your 2013 Commencement. To God be the glory for another wonderful rainy day. My sisters sometimes say to me, I have some tendency that is a little bit leaning towards crazy. So, I read stuff. People do not go on websites and read negative things except they have a little mental issue. So, as I was preparing for this commencement, something took me to Barnard website. And there was this article, ‚ÄúWhy Leymah Gbowee Commencement Speaker?‚Äù And then after reading part of the article, I usually would just skip through and go down to the comments. Trust me, you all did well, as compared to some of the sites that I go on. But one of the comments that I like, because this site is BW-O-G, and it said, ‚ÄúHow awesome,‚Äù that was the comment, ‚Äúfor a G-B-O-W-E-E, to be speaking at Barnard, on, and then we‚Äôre talking about her on this site, B-W-O-G. So, if you switch it around, except for the W-E that is my last name, but you were very nice to me. I have been asked to send you off with some words of wisdom. I‚Äôll do my best on the wisdom part. Words you will definitely get. I ask you graduates to kindly focus for a moment, forget the parties afterwards. Forget the presents that are awaiting you out there, and just journey with me, briefly, on the term, ‚ÄúStep out of the shadows.‚Äù And most times when I‚Äôm speaking at commencements or speaking with girls or women, I tend to put on something that will cause you ‚Äì even if you forgot what I said, to remember me. Unfortunately, today, I don‚Äôt have one of my big head gears that will make you remember me, but please try to remember my pretty 41-year-old face. And I‚Äôm donning and 18-year old hair style. So if you forget anything I say, don‚Äôt forget, she had a hairstyle like her daughter. Many years ago, I met an old woman. Her name was Krubo Pewee. She was quite poor, and lived in a shanty rundown home, but she had an air of confidence and independence. She walked with her shoulders up. Curiosity actually led me to seek this woman out. Every time I visited her, I would leave her some cash for food and medication, pitying her condition. She always hesitated taking the money from me. I would have to urge her before she reached out to take. One day, after several months of visitation and friendship, I handed her some money, and she said, thank you, but no thanks. She said, Leymah, I‚Äôm not one of those people to take money or to always take from people. I like giving back when I take. I‚Äôm a business woman. I love to watch my money grow, and I love to serve people through my business. If you want to do me a favor, give me a loan, so that I can restart my business. I asked her how much do you want? She said, 200 US dollars. In Liberian money, that is about $14,000. I took $250 and gave to her. Six months later, I went back to her tiny village. I saw a large kiosk, like a shop, rice, vegetables, and other provisions. I was shocked, but elated at the same time. She was more talkative, more relaxed, and we went on chatting about different things. As we talked, she asked about my children. And I told her about the headache of children being far away in school, and having to send money from Africa to the US, and she said ‚Äì I did that too. Of course I was shocked. You send money to the US? She said, yes. In the early 70s, my brother got a scholarship as an aircraft maintenance engineer from Liberia. And this scholarship only paid his fees. So, I had to send him money every month. So, I used to go and do bank drafts. Those were the days long before Western Unions or Money Grams. We talked about different things, and she revealed to me that from that kiosk, the previous one she had was what she used to educate that engineer, an IT consultant, a professional nurse, a community activist, and many more children of her relatives, siblings, and her own children. Again, I was shocked. Here is this woman, poor, sad, living in a shanty home, talking about all of these great people that she had educated. But as we continued the conversation, I said, but you‚Äôve done well to do all of this, and she would not for one moment take any credit for educating those individuals. She referred to herself as a shadow. A shadow, what the shadow does, according to her, is accompany you. It is never active. It doesn‚Äôt feed or clothe you. I told myself, a concept of her role in these people‚Äôs lives was wrong, but who was I to argue with a 76-year-old woman? Shadow does nothing. And as I drove away from that place, I kept thinking about how she referred to herself. And it dawned on me that this is how all over the world, women think. They do a lot of the work, but they never really take any credit for what they do. Their roles in the success or the successes of all of the different things, they always try to keep in the shadows. Growing up, most times as young women and as girls, regardless of where you come from we are socialized as women to be humble. In very extreme cases, be seen and never heard. In some cases, walk on tiptoes. For many years, I heard the phrase, ‚ÄúAct like a lady.‚Äù To sum it all up, we are expected to live our lives in the shadows, but we are also told to contribute our quota to the growth and development of the world. I have a four-year-old who is going on 55, and she constantly comes back from my parents‚Äô house, and says, Momma, Grandma said, ‚ÄúGirls don‚Äôt jump up and down.‚Äù And then I say to her, ‚ÄúMok, Momma says, jump up and down as much as you want!‚Äù Grandma says, ‚ÄúGood girls should read their books and be quite.‚Äù And then I say, ‚ÄúMok Momma says, good girls should read their books and tell the world what they‚Äôve read.‚Äù The contradictions of our lives as women, is confusing for me as an activist, sometimes. Sometimes, it‚Äôs enraging, and other times, it‚Äôs a little bit entertaining. A few months ago, I dared to speak up against the current regime. One of my uncles is a minister in this current regime. And he called my dad, and this is the entertaining part. Why can‚Äôt you control your daughter? And my dad said to him, ‚ÄúShe‚Äôs your niece. You go and control her.‚Äù But between the two men who was supposed to be controlling me, no one dare come to control me. We are told, for those of us who frequent international conferences and meetings, this is the decade of the women. This is followed by local and international proclamations on the rights of women and girls. These proclamations, in my opinion, are made to get us to put our best foot forward; get our brains working, and other instances get our well-manicured nails dirty. However, we‚Äôve seen also many examples of the reality of our situation. For in this country, women can join the military, but until recently, could not engage in active combat. My interpretation was that we are not to be put up front. Our roles are to be positioned, uniquely, in the shadows. In many other part of the world, including my own country Liberia, it is a struggle to convince fathers, and sometimes mothers that their daughters are worthy of being in school, and not in the shadows of the home. The story of Malala took the world by storm. This is another example. In college, many of you spend four years, especially in a women's college, listening to the rhetoric of the world, rhetoric that we hear at all international meetings about women's roles, responsibilities, and rights. The real world, ladies, will teach you as it is still teaching me that it will never be handed down to you on flower beds of ease as my mother called it, or on a silver platter. You have to challenge, in most cases, keep your hand up, in other cases, and in some cases, break protocol if you are to step out of the shadows. You were also taught some of the stories of great women, women who have left great legacies, Harriet Tubman enslaved, mildly epileptic, Black, and a woman. Those were all qualities, and reasons for her to remain in the shadows. She refused to do so. She engaged one cause after the other. Susan B. Anthony, women's rights activist, freedom fighter, she refused to be in the shadows. She spoke up in her lifetime about the inequalities between men and women, and freedom for those enslaved. Her earlier fear of public speaking never hindered her from stepping out of the shadows. These are just two examples of women of old in your context. Today there are many more that we could cite. The lessons these women have taught, and are still teaching us is that we must learn, decide, and fight to break out of the shadows; break out about your pains. I just came back from Libya where I heard horrid stories, horrible stories about rape and abuse during the revolution, and I was told the story of this young lady who was brutally raped. Her brothers locked her up, and because for them, her pain is to be kept in the shadows of their home, she broke free; ran away. They tracked her down, and killed her because she was to remain in the shadows. We went to this huge conference, and one of those young women who have also been in the shadows stepped out, and said, I want to speak about my rape. She came, covered in black, standing in that room that I called 98.2% of men, and told her story of how she was kept in a room with 80 other women raped daily, abused daily. The men in that room hung their head. I stood up, and applauded her because she refused to stay in the shadows of her pains. Don't stay in the shadows. Refuse to stay in the shadow. Break out about your dreams. Break out about your passion that you have for changing the world. Break out about how you feel about things. Never hold back. Refuse to be in the shadows as you step out into this life. Don't be shy no matter how crazy it seems to you. That crazy idea may just be the solution for some crazy global or local problem. From 1989 until 2003, the women of Liberia were also in the shadows. However, in 2003, tired of being used, and misused by over-drug militias, we stepped out to front the demons of militarism and violence. We refused to allow our bodies to be used anymore. We knew we would die, but we refused to allow our legacies to be ‚Äúthey died without trying.‚Äù We stepped out of the darkness of victimization, and into the light of activism and peace. We changed the global perception of Liberia being The Land of Child Soldiers to being The Land of Women in White. Today, the peace that we strived for in Liberia has been translated into many empowerment, and refusal to be seen, and not heard. Community women are demanding their rights, demanding justice for perpetrators of crimes against women, and demanding the provisions of basic social services. We, as women of Liberia, are also demanding recognition for our contributions to the growth and development of our nation. Sheryl Sandberg, a good friend, and someone who I stand behind because she came ahead of me to Barnard writes in her book, Lean In, that women should step out, and unashamedly claim their spaces in their professional career striving to be out and on top. This, my dear ladies, can only happen if you step out of the shadows. I received a t-shirt once that read, ""Good girls never make history."" I love it because it encourages me to remain in the light, and never step back into the shadows. So, I started with the story of Krubo Peewee in August of 2013, one of those she educated died, the aviation engineer. I accompanied her to the family meeting planning the burial. The entire time no one acknowledged her, or recognized her. She sat in the back of the meeting sobbing quietly still hiding in the shadows somewhat hopeful that someone will recognize the role she played in this man's life. It never happened. On the day of the funeral, I went along with her. We sat in the church, and one-after-the-other people came, and paid tribute, and attributed his successes to one thing or the other; never the poor woman in the shanty run-down house. Finally, the pastor announced, if there were no more tributes, they will continue with the other aspects of the program. I was sitting, and screaming in my head, go for it, Krubo! Stand up. Say something. Step out of the shadows. And, as if she could hear my mental scream, she stood up, straightened her shoulders, and walked up to the podium. Here lies a man I saw so much ability in. I live my life through him. I did not go to school because our parents married me off early. And, because I could make money, I sent him to school, and she went on to talk about her brother, and everything she did. Afterwards, she turned to his children and his widow, and she said to them, ‚ÄúIt's always good to recognize someone, anyone, regardless of their physical appearance when they have contributed to your success.‚Äù As she walked out of the church, I followed and went, yes! Distinguished graduates, as you journey through life, refuse to hide. Each and every one of you has unique skills and qualities that the world needs. Being in the shadow will continue to keep our dark world, darker. If all of you decide, or decided that this life you will step out, and do exactly what we need to do, you'll make the world a better place. Like Krubo Peewee, you may be forced to step out of the shadows. No matter how you decide to do so, always remember that stepping out of the shadows will ensure, your stepping out, will ensure that some girl will also find the strength to step out. Many years ago, I made that decision. Four children, dirt broke, dirt poor, only two underwear, until today, I am traumatized, so I buy underwear like a crazy person. I have to say that. Dirt poor, I went back to school, and I sat in my college classroom for three months, and never said a word. Every time someone raised their hand, and said something, I said to myself, I could have said it better. On this fateful day, I got this philosophy assignment, and I put my all into that assignment, went back, and presented my papers, psychology; not philosophy, went back, presented my paper to my professor, and when he brought it, I had an F. I looked at the paper, and something was telling me step out of the shadows. As long as you remain in the shadows, you will continue to receive F. I sat there, looked at that paper, looked, and thought, and looked, and thought, and mustered the courage; mustered the bravery. After class, walked up to the professor sweating like a goat during wintertime, sweating, really sweating profusely, shaking like a leaf, and I said to him, ‚ÄúSir, you miss-graded my paper.‚Äù He looked at me with a stern face, and said, because this is my first time speaking to this man in three months, ‚ÄúAre you sure‚Äù? And, I said, ‚ÄúYes.‚Äù I feel because I have never spoken up in class, you give me an F; you give me an F without reading my paper. And then, he took it away from me, and said, if, and only in Africa the professor will do that, if you're telling a lie, you will be in trouble with me, and the only thing that rang in my head, he who is down, fear no fall. He went back, and brought that paper on Monday, and I got an A+. He saw the name, and never heard the voice, and thought that name is equivalent to F. As you step out, please, you're more than F. You're more than D. You're more than C. You're even more than B. I tell my children the alphabet starts from A, and that's what God has put in every woman in this world. You are an A. Refuse to be in the shadows. Because as you remain in the shadows, someone will miss-grade you, miss, or underpay you, misuse, abuse you. Refuse to remain in the shadow. Step out of the shadow. And you decide to step out of the shadow, just in case some father, brother, sister, mother, or former professor tries to tell you that a girl has never done this before, remind them that a woman came all the way from Africa to tell us, the world is upside down. Things are not what they used to be before. The Black man is one of the best golfers. White boys are playing basketball very well. Two women are president of Africa, and a White man and a Black man and his family now lives in The White House. Step out of the shadows, and be the best God created you to be. Congratulations, students. Thank you, parents. Well done, faculty. God bless us all. Thank you.",Barnard College - Columbia University official website -linda Thomas Greenfield ,"Thank you, Mr. President. Thank you for convening us to commemorate this important day. And I thank the SecretaryGeneral, Madam High Commissioner, and Dr. Iweala, for your leadership in pressing us all to do more toward the elimination of racial discrimination, wherever and by whomever. This meeting ‚Äì this commemoration ‚Äì is personal to me. I am a person of African descent. But more importantly, I am a descendant of slaves. My great grandmother Mary Thomas, born in 1865, was the child of a slave. This is just three generations back from me. I grew up in the segregated South. I was bused to a segregated school, and on weekends, the Ku Klux Klan burned crosses on lawns in our neighborhood. When I was in high school, I was asked by a little girl, for whom I babysat, if I was an N-word because her dad had used that word for me. I know the ugly face of racism. I lived racism. I have experienced racism. And I survived racism. And through this process, I learned a simple truth: Racism is not the problem of the person who experiences it. Those of us who experience racism cannot, and should not, internalize it, despite the impact that it can have on our everyday lives. We must face it down, every time, no matter whom it‚Äôs directed towards. Racism is the problem of the racist. And it is the problem of the society that produces the racist. And in today‚Äôs world, that is every society. And in so many of our communities and countries, racism is endemic. It‚Äôs built in, like a rot in a frame. And it remains, and it festers, and it spreads because many of those in charge allow it to. Others look away and pretend it‚Äôs not there. But like a cancer, if ignored, it grows. Today, we commemorate our joint commitment to end all racial discrimination. And we take stock of our efforts during the midterm review of the Decade of People of African Descent. In America, conducting that review requires a reckoning ‚Äì a reckoning with our dark history of chattel slavery. Four-hundred-and-two years ago, African slaves were forced onto the shores of the colony of Virginia. Two years ago, the 1619 Project brought attention to this anniversary, and put the consequences of slavery, and the contributions of Black Americans, back at the center of our history and of our national narrative. As the project detailed, slavery is the original sin of America. It‚Äôs weaved white supremacy and black inferiority into our founding documents and principles. The Legacy Museum in Alabama traces this history, and if you‚Äôve not been there, I encourage you all to take a trip. Its exhibits draw a direct line from slavery to lynchings to segregation to mass incarceration and testify to this terrible history and the impact it is having on our people today. But even though slavery is our original sin, America is not the original source of slavery. Others share this shame with us. Slavery has existed in every corner of the globe. Africans enslaved fellow Africans long before the American colonists existed. And sadly, in many places around the world, slavery still exists today. As the scholar Isabel Wilkerson argues, humans in all contexts have ranked human value, pitting the presumed supremacy of one group against the presumed inferiority of others. In America, that takes many forms. Chief among them: our legacy of white supremacy. This year, the senseless killing of George Floyd, Breonna Taylor, and so many other Black Americans sparked a reckoning with racial justice, a movement that spread across the world: Black Lives Matter. And because Black Lives Matter, we need to dismantle white supremacy at every turn. This means looking at other kinds of hate, too. The FBI has reported a spike in hate crimes over the past three years ‚Äì particularly against Latino Americans, Sikhs, Muslim Americans, Jewish Americans, and immigrants. The most recent data shows hate crimes rising to a level not seen in over a decade. And that doesn‚Äôt even capture the bullying, discrimination, brutality, and violence that Asian Americans have faced since the outbreak of COVID-19. The mass shooting in Atlanta is only the latest example of this horror. At President Biden‚Äôs direction, we are flying our flag at half-staff at the U.S. Mission to the United Nations, to honor the victims of this terrible, senseless tragedy. It is so important we stand together ‚Äì we stand unified ‚Äì against this scourge. In unity, we have strength. But divisions and misperceptions about each other work against all of us. We also need to recognize that racism is far from unique in America. Across four decades and four continents in the Foreign Service, I experienced racism in countless international contexts, from overly zealous searches at airports, to police racially profiling my son, to being made to wait behind white patrons for a table at a restaurant. Racism was and continues to be a daily challenge wherever we are. And for millions, it‚Äôs more than a challenge. It‚Äôs deadly. Like in Burma, where Rohingya and others have been oppressed, abused, and killed in staggering numbers. Or in China, where the government has committed genocide and crimes against humanity against Uyghurs and members of other ethnic and religious minority groups in Xinjiang. The prevalence, and pervasiveness, of racial discrimination might make the situation look hopeless. But let me be clear: I remain hopeful. I am hopeful because I have seen how communities and countries can enact change. And I have experienced that progress in my own lifetime. Personally, I am just one example of what hope and strength can do. After all, this descendant of slaves is before you today as the U.S. Representative to the United Nations. The first chapter of my life story ‚Äì born in poverty to uneducated parents ‚Äì this could not have been predicted. So I ask, what can we do to promote change and keep hope alive for victims of racism? We can‚Äôt control the hate in people‚Äôs heart. But we can change the rules that give them license. That‚Äôs how I‚Äôm sitting here. It‚Äôs why we were able to welcome Vice President Kamala Harris to the UN this week. It‚Äôs why President Biden‚Äôs cabinet is the most diverse in history and includes the first Native American named to a cabinet post. We can make our communities, and our governments, reflect our highest aspirations ‚Äì even if some individuals still fall short. We can act. And in the Biden-Harris administration, we are doing just that. In the first 60 days, the President has made this a priority: from redressing racial discrimination in housing, to ending private prisons that warehouse young black and brown men, to respecting the sovereignty of Native American tribes, to combatting xenophobia and discrimination against Asians, Asian Americans, and Pacific Islanders. The Biden-Harris Administration also recognizes how the COVID-19 pandemic and economic crisis has been disproportionately damaging to members of racial and ethnic minorities. So, we have taken steps, like providing emergency relief funds, increasing access to nutritious food, and passing* federal student loan payments, that we know will particularly help Black and brown communities. To be clear, this is just the beginning. Ending racial discrimination, particularly in our criminal justice system, will be an ongoing top priority for the President, and for the entire Biden-Harris Administration. And we ask that other countries join us. We call for all countries to ratify and implement the International Convention on the Elimination of All Forms of Racial Discrimination. After all, this is about shaping the future. It‚Äôs shaping the future we want for our children, and our grandchildren, and their grandchildren. Already, they are demanding we do better. They are coming up with new ideas and they‚Äôre pushing for progressive action. They‚Äôre asking more from their politicians and their governments. And they‚Äôre in the streets, marching for charge. They say that ‚ÄúBlack Lives Matter.‚Äù Because they do. They chant: ‚ÄúThis is what democracy looks like.‚Äù Because it is. This is the American way. We have flaws. Deep, serious flaws. But we talk about them. We work to address them. And we press on, in hopes that we can leave the country better than we found it. We can do the same on a multilateral scale. Let us expose the racism and racial discrimination endemic in every society, around the globe. Let us press forward, to root out that discrimination and remove the rot from our foundations. And on this day dedicated to ending racial discrimination, as our flags fly at half-staff, let us leave our children a less hateful, more hopeful world. Let us give them a future. A future without fear. A future without violence. That is the legacy that I hope they can inherit. Thank you.",United States mission to the United Nation -Malala Yousafzai,"In the name of God, the most beneficent, the most merciful. - -Honorable UN Secretary General Mr Ban Ki-moon, respected president of the General Assembly Vuk Jeremic, honorable UN envoy for global education Mr Gordon Brown, respected elders and my dear brothers and sisters: Assalamu alaikum. - -Today is it an honor for me to be speaking again after a long time. Being here with such honorable people is a great moment in my life and it is an honor for me that today I am wearing a shawl of the late Benazir Bhutto. I don't know where to begin my speech. I don't know what people would be expecting me to say, but first of all thank you to God for whom we all are equal and thank you to every person who has prayed for my fast recovery and new life. I cannot believe how much love people have shown me. I have received thousands of good wish cards and gifts from all over the world. Thank you to all of them. Thank you to the children whose innocent words encouraged me. Thank you to my elders whose prayers strengthened me. I would like to thank my nurses, doctors and the staff of the hospitals in Pakistan and the UK and the UAE government who have helped me to get better and recover my strength. - -I fully support UN Secretary General Ban Ki-moon in his Global Education First Initiative and the work of UN Special Envoy for Global Education Gordon Brown and the respectful president of the UN General Assembly Vuk Jeremic. I thank them for the leadership they continue to give. They continue to inspire all of us to action. Dear brothers and sisters, do remember one thing: Malala Day is not my day. Today is the day of every woman, every boy and every girl who have raised their voice for their rights. - -There are hundreds of human rights activists and social workers who are not only speaking for their rights, but who are struggling to achieve their goal of peace, education and equality. Thousands of people have been killed by the terrorists and millions have been injured. I am just one of them. So here I stand. So here I stand, one girl, among many. I speak not for myself, but so those without a voice can be heard. Those who have fought for their rights. Their right to live in peace. Their right to be treated with dignity. Their right to equality of opportunity. Their right to be educated. - -Dear friends, on 9 October 2012, the Taliban shot me on the left side of my forehead. They shot my friends, too. They thought that the bullets would silence us, but they failed. And out of that silence came thousands of voices. The terrorists thought they would change my aims and stop my ambitions. But nothing changed in my life except this: weakness, fear and hopelessness died. Strength, power and courage was born. - -I am the same Malala. My ambitions are the same. My hopes are the same. And my dreams are the same. Dear sisters and brothers, I am not against anyone. Neither am I here to speak in terms of personal revenge against the Taliban or any other terrorist group. I am here to speak for the right of education for every child. I want education for the sons and daughters of the Taliban and all the terrorists and extremists. I do not even hate the Talib who shot me. Even if there was a gun in my hand and he was standing in front of me, I would not shoot him. This is the compassion I have learned from Mohammed, the prophet of mercy, Jesus Christ and Lord Buddha. This the legacy of change I have inherited from Martin Luther King, Nelson Mandela and Mohammed Ali Jinnah. - -This is the philosophy of nonviolence that I have learned from Gandhi, Bacha Khan and Mother Teresa. And this is the forgiveness that I have learned from my father and from my mother. This is what my soul is telling me: be peaceful and love everyone. - -Dear sisters and brothers, we realize the importance of light when we see darkness. We realize the importance of our voice when we are silenced. In the same way, when we were in Swat, the north of Pakistan, we realized the importance of pens and books when we saw the guns. The wise saying, ""The pen is mightier than the sword."" It is true. The extremists are afraid of books and pens. The power of education frightens them. They are afraid of women. The power of the voice of women frightens them. This is why they killed 14 innocent students in the recent attack in Quetta. And that is why they kill female teachers. That is why they are blasting schools every day because they were and they are afraid of change and equality that we will bring to our society. And I remember that there was a boy in our school who was asked by a journalist why are the Taliban against education? He answered very simply by pointing to his book, he said, ""a Talib doesn't know what is written inside this book."" - -They think that God is a tiny, little conservative being who would point guns at people's heads just for going to school. These terrorists are misusing the name of Islam for their own personal benefit. Pakistan is a peace loving, democratic country. Pashtuns want education for their daughters and sons. Islam is a religion of peace, humanity and brotherhood. It is the duty and responsibility to get education for each child, that is what it says. Peace is a necessity for education. In many parts of the world, especially Pakistan and Afghanistan, terrorism, war and conflicts stop children from going to schools. We are really tired of these wars. Women and children are suffering in many ways in many parts of the world. - -In India, innocent and poor children are victims of child labor. Many schools have been destroyed in Nigeria. People in Afghanistan have been affected by extremism. Young girls have to do domestic child labor and are forced to get married at an early age. Poverty, ignorance, injustice, racism and the deprivation of basic rights are the main problems, faced by both men and women. - -Today I am focusing on women's rights and girls' education because they are suffering the most. There was a time when women activists asked men to stand up for their rights. But this time we will do it by ourselves. I am not telling men to step away from speaking for women's rights, but I am focusing on women to be independent and fight for themselves. So dear sisters and brothers, now it's time to speak up. So today, we call upon the world leaders to change their strategic policies in favor of peace and prosperity. We call upon the world leaders that all of these deals must protect women and children's rights. A deal that goes against the rights of women is unacceptable. - -We call upon all governments to ensure free, compulsory education all over the world for every child. We call upon all the governments to fight against terrorism and violence. To protect children from brutality and harm. We call upon the developed nations to support the expansion of education opportunities for girls in the developing world. We call upon all communities to be tolerant, to reject prejudice based on caste, creed, sect, color, religion or agenda to ensure freedom and equality for women so they can flourish. We cannot all succeed when half of us are held back. We call upon our sisters around the world to be brave, to embrace the strength within themselves and realize their full potential. - -Dear brothers and sisters, we want schools and education for every child's bright future. We will continue our journey to our destination of peace and education. No one can stop us. We will speak up for our rights and we will bring change to our voice. We believe in the power and the strength of our words. Our words can change the whole world because we ware all together, united for the cause of education. And if we want to achieve our goal, then let us empower ourselves with the weapon of knowledge and let us shield ourselves with unity and togetherness. - -Dear brothers and sisters, we must not forget that millions of people are suffering from poverty and injustice and ignorance. We must not forget that millions of children are out of their schools. We must not forget that our sisters and brothers are waiting for a bright, peaceful future. - -So let us wage, so let us wage a glorious struggle against illiteracy, poverty and terrorism, let us pick up our books and our pens, they are the most powerful weapons. One child, one teacher, one book and one pen can change the world. Education is the only solution. Education first. Thank you.",Iowa state university archives -Michelle bachelet,"Distinguished Ministers, -Excellencies, -Colleagues and friends, - -I am very pleased to be here, and I thank the Core Group for organizing today’s event, especially our co-chairs, Argentina and the Netherlands. - -It is fundamental to the cause of human rights that we promote equality, and the protection of all people from discrimination and violence. - -It should be obvious that there are many different ways to be a human being. We need to respect and embrace these differences -- not criminalise them, not attack people, not deprive them of equal rights or the protection of the law, just because they are seen as “different”. - -The struggle for the rights of LGBTI people is a core part of the human rights struggle. - -And yet in many parts of the world, members of the LGBTI community continue to be the targets of brutal attacks, many of which are left unpunished. - -It is essential that we defend and protect the LGBTI community, from every kind of violence and discrimination. - -There should be nothing “controversial” about stopping people being murdered, or executed by agents of the State, simply because of who they are or whom they love. Tackling extreme violence does not require new norms. - -Seventy years ago, the Universal Declaration of Human Rights proclaimed that everyone, without distinction “has the right to life, liberty and security of the person.” When there is a pattern of hate-motivated violence – one, for example, based on gender, sexual orientation or gender identity – and the State does not act to prevent, and effectively address, those attacks, it is failing to live up to that obligation. - -Excellencies, - -In a number of countries, there have been patterns of hate-motivated killings against lesbian, gay, bi and trans people by private actors and, sometimes, by local security forces or non-State armed groups. In some countries, trans women, in particular, have been at risk of such killings. - -Worse still, when the victims of attacks seek protection, they are frequently subjected to intimidation and abuse, including from police and justice officials. And most countries do not track homophobic and transphobic crimes. - -The result is impunity. Too many victims go without recognition, remedy or justice. Too many perpetrators are free to strike again, undeterred by the prospect of rule of law. - -Intersex people, too, face violence. There have been reports of the killing of young intersex babies. Some intersex children are subjected to harmful practices in medical settings. And very few States are taking action to protect intersex children from such harm. - -LGBTI people are entitled to equal protection, and the same rights as everyone else. - -I am encouraged by the fact that an unprecedented number of countries are now committed to taking action to prevent and address killings and violence against LGBTI people. - -But we need for all States to step up their efforts to address these crimes; to prevent extrajudicial, summary or arbitrary executions; and to protect all people, without discrimination. - -Excellencies, - -Tragically, the issue is not only the inadequate response by the State to violence against LGBTI people. - -In seven countries, national or provincial laws provide for the execution of people convicted of acts related to homosexuality. I want to make clear that use of the death penalty in these circumstances is in complete violation of fundamental rights. - -More than 70 countries criminalize consensual same sex relationships, and also criminalize transgender people based on their appearance. These laws subject LGBT people to long prison sentences, and in some cases physical punishment. They also implicitly encourage prejudice, hatred and violence. - -But laws can change. - -In my country, Chile, following the brutal torture and murder of a young gay man in a Santiago park, six years ago, a very powerful and moving public discussion took place about the need to challenge hatred and violence towards the LGBT community. A bill, blocked for years in Parliament, was adopted – making it easier to punish homophobic and transphobic crimes. - -In India, we have just seen a landmark decision by the Supreme Court, decriminalising same-sex relationships. - -This important discussion is taking place all over the world. Not only in Europe and North America – it is moving forward in Africa, in Asia, in the Americas, in the Caribbean and in the Pacific. - -I welcome these vital changes. We need to see more countries taking steps to bring their laws and practices in line with the fundamental equality of all their people. - -But we need more. - -We need to change minds. - -At the core of killings and violence against LGBTI community is prejudice and hate. We will only prevent these crimes if we are brave enough to address these factors, across society. - -I am proud of the work my Office is doing to open people’s hearts and minds through its global Free & Equal campaign. This is one of the biggest UN public information initiatives ever undertaken, and it has reached hundreds of millions of people around the world. - -We also need education, education, education. Respect for diversity – including in relation to LGBTI people – should be reflected in school curricula and reinforced through effective public information campaigns. - -We need the business community, religious leaders, celebrities and the media to play a positive role. - -Excellencies, - -I believe that profound, positive change is possible. I have seen it in my own lifetime. It can be done. - -With your help, and the help of many others, we can prevent killings, violence, humiliation and fear in our LGBTI community. - -We can conquer hate. - -Thank you",United Nations office of high commisioner official website \ No newline at end of file diff --git a/content/tutorial-plotting-fractals.md b/content/tutorial-plotting-fractals.md index e97b0cbe..beb64e6d 100644 --- a/content/tutorial-plotting-fractals.md +++ b/content/tutorial-plotting-fractals.md @@ -110,7 +110,7 @@ mesh = x + (1j * y) # Make mesh of complex plane Now we will apply our function to each value contained in the mesh. Since we used a universal function in our design, this means that we can pass in the entire mesh all at once. This is extremely convenient for two reasons: It reduces the amount of code needed to be written and greatly increases the efficiency (as universal functions make use of system level C programming in their computations). -Here we plot the absolute value (or modulus) of each element in the mesh after one “iteration” of the function using a [**3D scatterplot**](https://matplotlib.org/2.0.2/mpl_toolkits/mplot3d/tutorial.html#scatter-plots): +Here we plot the absolute value (or modulus) of each element in the mesh after one “iteration” of the function using a [**3D scatterplot**](https://matplotlib.org/stable/users/explain/toolkits/mplot3d.html#scatter-plots): ```{code-cell} ipython3 output = np.abs(f(mesh)) # Take the absolute value of the output (for plotting) @@ -301,14 +301,14 @@ For example, setting $c = \frac{\pi}{10}$ gives us a very elegant cloud shape, w ```{code-cell} ipython3 output = julia(mesh, c=np.pi/10, num_iter=20) -kwargs = {'title': 'f(z) = z^2 + \dfrac{\pi}{10}', 'cmap': 'plasma'} +kwargs = {'title': r'f(z) = z^2 + \dfrac{\pi}{10}', 'cmap': 'plasma'} plot_fractal(output, **kwargs); ``` ```{code-cell} ipython3 output = julia(mesh, c=-0.75 + 0.4j, num_iter=20) -kwargs = {'title': 'f(z) = z^2 - \dfrac{3}{4} + 0.4i', 'cmap': 'Greens_r'} +kwargs = {'title': r'f(z) = z^2 - \dfrac{3}{4} + 0.4i', 'cmap': 'Greens_r'} plot_fractal(output, **kwargs); ``` @@ -334,7 +334,7 @@ def mandelbrot(mesh, num_iter=10, radius=2): ```{code-cell} ipython3 output = mandelbrot(mesh, num_iter=50) -kwargs = {'title': 'Mandelbrot \ set', 'cmap': 'hot'} +kwargs = {'title': 'Mandelbrot \\ set', 'cmap': 'hot'} plot_fractal(output, **kwargs); ``` @@ -370,8 +370,6 @@ for deg, ax in enumerate(axes.ravel()): diverge_len = general_julia(mesh, f=power, num_iter=15) ax.imshow(diverge_len, extent=[-2, 2, -2, 2], cmap='binary') ax.set_title(f'$f(z) = z^{degree} -1$') - -fig.tight_layout(); ``` Needless to say, there is a large amount of exploring that can be done by fiddling with the inputted function, value of $c$, number of iterations, radius and even the density of the mesh and choice of colours. @@ -419,7 +417,7 @@ p.deriv() ```{code-cell} ipython3 output = newton_fractal(mesh, p, p.deriv(), num_iter=15, r=2) -kwargs = {'title': 'f(z) = z - \dfrac{(z^8 + 15z^4 - 16)}{(8z^7 + 60z^3)}', 'cmap': 'copper'} +kwargs = {'title': r'f(z) = z - \dfrac{(z^8 + 15z^4 - 16)}{(8z^7 + 60z^3)}', 'cmap': 'copper'} plot_fractal(output, **kwargs) ``` @@ -443,7 +441,7 @@ def d_tan(z): ```{code-cell} ipython3 output = newton_fractal(mesh, f_tan, d_tan, num_iter=15, r=50) -kwargs = {'title': 'f(z) = z - \dfrac{sin(z)cos(z)}{2}', 'cmap': 'binary'} +kwargs = {'title': r'f(z) = z - \dfrac{sin(z)cos(z)}{2}', 'cmap': 'binary'} plot_fractal(output, **kwargs); ``` @@ -475,7 +473,7 @@ We will denote this one 'Wacky fractal', as its equation would not be fun to try ```{code-cell} ipython3 output = newton_fractal(small_mesh, sin_sum, d_sin_sum, num_iter=10, r=1) -kwargs = {'title': 'Wacky \ fractal', 'figsize': (6, 6), 'extent': [-1, 1, -1, 1], 'cmap': 'terrain'} +kwargs = {'title': 'Wacky \\ fractal', 'figsize': (6, 6), 'extent': [-1, 1, -1, 1], 'cmap': 'terrain'} plot_fractal(output, **kwargs) ``` @@ -550,7 +548,7 @@ def accident(z): ```{code-cell} ipython3 output = general_julia(mesh, f=accident, num_iter=15, c=0, radius=np.pi) -kwargs = {'title': 'Accidental \ fractal', 'cmap': 'Blues'} +kwargs = {'title': 'Accidental \\ fractal', 'cmap': 'Blues'} plot_fractal(output, **kwargs); ``` diff --git a/content/tutorial-static_equilibrium.md b/content/tutorial-static_equilibrium.md index 0e8b82f7..29164be9 100644 --- a/content/tutorial-static_equilibrium.md +++ b/content/tutorial-static_equilibrium.md @@ -1,4 +1,5 @@ --- +short_title: Static Equilibrium jupytext: text_representation: extension: .md @@ -77,7 +78,7 @@ This defines `forceA` as being a vector with magnitude of 1 in the $x$ direction It may be helpful to visualize these forces in order to better understand how they interact with each other. Matplotlib is a library with visualization tools that can be utilized for this purpose. -Quiver plots will be used to demonstrate [three dimensional vectors](https://matplotlib.org/3.3.4/gallery/mplot3d/quiver3d.html), but the library can also be used for [two dimensional demonstrations](https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.quiver.html). +Quiver plots will be used to demonstrate [three dimensional vectors](https://matplotlib.org/gallery/mplot3d/quiver3d.html), but the library can also be used for [two dimensional demonstrations](https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.quiver.html). ```{code-cell} fig = plt.figure() @@ -190,7 +191,7 @@ The pole does not move, so it must apply a reaction force. The pole also does not rotate, so it must also be creating a reaction moment. Solve for both the reaction force and moments. -Lets say a 5N force is applied perpendicularly 2m above the base of the pole. +Let's say a 5N force is applied perpendicularly 2m above the base of the pole. ```{code-cell} f = 5 # Force in newtons @@ -263,7 +264,7 @@ print("Reaction moment =", M) ``` ### Another Example -Let's look at a slightly more complicated model. In this example you will be observing a beam with two cables and an applied force. This time you need to find both the tension in the cords and the reaction forces of the beam. *(Source: [Vector Mechanics for Engineers: Statics](https://www.mheducation.com/highered/product/vector-mechanics-engineers-statics-beer-johnston/M9780077687304.html), Problem 4.106)* +Let's look at a slightly more complicated model. In this example you will be observing a beam with two cables and an applied force. This time you need to find both the tension in the cords and the reaction forces of the beam. *(Source: [Vector Mechanics for Engineers: Statics and Dynamics](https://www.mheducation.com/highered/product/Vector-Mechanics-for-Engineers-Statics-and-Dynamics-Beer.html), Problem 4.106)* ![image.png](_static/problem4.png) @@ -387,5 +388,5 @@ This same process can be applied to kinetic problems or in any number of dimensi ### References -1. [Vector Mechanics for Engineers: Statics (Beer & Johnston & Mazurek)](https://www.mheducation.com/highered/product/vector-mechanics-engineers-statics-beer-johnston/M9780077687304.html) +1. [Vector Mechanics for Engineers: Statics and Dynamics (Beer & Johnston & Mazurek & et al.)](https://www.mheducation.com/highered/product/Vector-Mechanics-for-Engineers-Statics-and-Dynamics-Beer.html) 2. [NumPy Reference](https://numpy.org/doc/stable/reference/) diff --git a/content/tutorial-style-guide.md b/content/tutorial-style-guide.md index 24c6bef3..f0f49e52 100644 --- a/content/tutorial-style-guide.md +++ b/content/tutorial-style-guide.md @@ -1,4 +1,5 @@ --- +short_title: Style Guide jupytext: text_representation: extension: .md diff --git a/content/tutorial-svd.md b/content/tutorial-svd.md index 3a0b58cd..f7ab3c5c 100644 --- a/content/tutorial-svd.md +++ b/content/tutorial-svd.md @@ -1,4 +1,5 @@ --- +short_title: Linear Algebra on n-D arrays jupytext: text_representation: extension: .md @@ -35,15 +36,26 @@ After this tutorial, you should be able to: ## Content -In this tutorial, we will use a [matrix decomposition](https://en.wikipedia.org/wiki/Matrix_decomposition) from linear algebra, the Singular Value Decomposition, to generate a compressed approximation of an image. We'll use the `face` image from the [scipy.misc](https://docs.scipy.org/doc/scipy/reference/misc.html#module-scipy.misc) module: +In this tutorial, we will use a [matrix decomposition](https://en.wikipedia.org/wiki/Matrix_decomposition) from linear algebra, the Singular Value Decomposition, to generate a compressed approximation of an image. We'll use the `face` image from the [scipy.datasets](https://docs.scipy.org/doc/scipy/reference/datasets.html) module: ```{code-cell} -from scipy import misc +from scipy.datasets import face -img = misc.face() +img = face() ``` -**Note**: If you prefer, you can use your own image as you work through this tutorial. In order to transform your image into a NumPy array that can be manipulated, you can use the `imread` function from the [matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. Alternatively, you can use the [imageio.imread](https://imageio.readthedocs.io/en/stable/userapi.html#imageio.imread) function from the `imageio` library. Be aware that if you use your own image, you'll likely need to adapt the steps below. For more information on how images are treated when converted to NumPy arrays, see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. +```{note} +If you prefer, you can use your own image as you work through this tutorial. +In order to transform your image into a NumPy array that can be manipulated, you +can use the `imread` function from the +[matplotlib.pyplot](https://matplotlib.org/api/_as_gen/matplotlib.pyplot.html#module-matplotlib.pyplot) submodule. +Alternatively, you can use the +[imageio.imread](https://imageio.readthedocs.io/en/stable/_autosummary/imageio.v3.imread.html) +function from the `imageio` library. +Be aware that if you use your own image, you'll likely need to adapt the steps below. +For more information on how images are treated when converted to NumPy arrays, +see [A crash course on NumPy for images](https://scikit-image.org/docs/stable/user_guide/numpy_images.html) from the `scikit-image` documentation. +``` +++ @@ -76,7 +88,7 @@ First, let's check for the shape of the data in our array. Since this image is t img.shape ``` -The output is a [tuple](https://docs.python.org/dev/tutorial/datastructures.html#tut-tuples) with three elements, which means that this is a three-dimensional array. In fact, since this is a color image, and we have used the `imread` function to read it, the data is organized in three 2D arrays, representing color channels (in this case, red, green and blue - RGB). You can see this by looking at the shape above: it indicates that we have an array of 3 matrices, each having shape 768x1024. +The output is a [tuple](https://docs.python.org/dev/tutorial/datastructures.html#tut-tuples) with three elements, which means that this is a three-dimensional array. Since this is a color image, and we have used the `imread` function to read it, the data is organized as a 768×1024 grid of pixels, where each pixel contains 3 values representing color channels (red, green and blue - RGB). You can see this by looking at the shape, where the leftmost number corresponds to the outermost axis (image height), the middle number to the next axis (image width) and the rightmost number to the innermost axis (the color channels). Furthermore, using the `ndim` property of this array, we can see that @@ -91,7 +103,7 @@ img[:, :, 0] ``` From the output above, we can see that every value in `img[:, :, 0]` is an integer value between 0 and 255, representing the level of red in each corresponding image pixel (keep in mind that this might be different if you -use your own image instead of [scipy.misc.face](https://docs.scipy.org/doc/scipy/reference/generated/scipy.misc.face.html#scipy.misc.face)). +use your own image instead of [scipy.datasets.face](https://docs.scipy.org/doc/scipy/reference/generated/scipy.datasets.face.html)). As expected, this is a 768x1024 matrix: @@ -105,13 +117,19 @@ Since we are going to perform linear algebra operations on this data, it might b img_array = img / 255 ``` -This operation, dividing an array by a scalar, works because of NumPy's [broadcasting rules](https://numpy.org/devdocs/user/theory.broadcasting.html#array-broadcasting-in-numpy). (Note that in real-world applications, it would be better to use, for example, the [img_as_float](https://scikit-image.org/docs/stable/api/skimage.html#skimage.img_as_float) utility function from `scikit-image`). +This operation, dividing an array by a scalar, works because of NumPy's [broadcasting rules](https://numpy.org/devdocs/user/theory.broadcasting.html#array-broadcasting-in-numpy). + +```{tip} +In real-world applications, it may be better to use, for example, the +[img_as_float](https://scikit-image.org/docs/stable/api/skimage.html#skimage.img_as_float) +utility function from `scikit-image`. +``` You can check that the above works by doing some tests; for example, inquiring about maximum and minimum values for this array: ```{code-cell} -img_array.max(), img_array.min() +img_array.min(), img_array.max() ``` or checking the type of data in the array: @@ -134,23 +152,29 @@ It is possible to use methods from linear algebra to approximate an existing set +++ -**Note**: We will use NumPy's linear algebra module, [numpy.linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg), to perform the operations in this tutorial. Most of the linear algebra functions in this module can also be found in [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg), and users are encouraged to use the [scipy](https://docs.scipy.org/doc/scipy/reference/index.html#module-scipy) module for real-world applications. However, some functions in the [scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg) module, such as the SVD function, only support 2D arrays. For more information on this, check the [scipy.linalg Reference](https://docs.scipy.org/doc/scipy/reference/tutorial/linalg.html). +```{note} +We will use NumPy's linear algebra module, +[numpy.linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg), +to perform the operations in this tutorial. +Most of the linear algebra functions in this module can also be found in +[scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg), +and users are encouraged to use the [scipy](https://docs.scipy.org/doc/scipy/reference/index.html#module-scipy) +module for real-world applications. +However, some functions in the +[scipy.linalg](https://docs.scipy.org/doc/scipy/reference/linalg.html#module-scipy.linalg) +module, such as the SVD function, only support 2D arrays. +For more information on this, check the [scipy.linalg page](https://docs.scipy.org/doc/scipy/tutorial/linalg.html). +``` +++ -To proceed, import the linear algebra submodule from NumPy: - -```{code-cell} -from numpy import linalg -``` - In order to extract information from a given matrix, we can use the SVD to obtain 3 arrays which can be multiplied to obtain the original matrix. From the theory of linear algebra, given a matrix $A$, the following product can be computed: $$U \Sigma V^T = A$$ where $U$ and $V^T$ are square and $\Sigma$ is the same size as $A$. $\Sigma$ is a diagonal matrix and contains the [singular values](https://en.wikipedia.org/wiki/Singular_value) of $A$, organized from largest to smallest. These values are always non-negative and can be used as an indicator of the "importance" of some features represented by the matrix $A$. -Let's see how this works in practice with just one matrix first. Note that according to [colorimetry](https://en.wikipedia.org/wiki/Grayscale#Colorimetric_(perceptual_luminance-reserving)_conversion_to_grayscale), +Let's see how this works in practice with just one matrix first. Note that according to [colorimetry](https://en.wikipedia.org/wiki/Grayscale#Colorimetric_(perceptual_luminance-preserving)_conversion_to_grayscale), it is possible to obtain a fairly reasonable grayscale version of our color image if we apply the formula $$Y = 0.2126 R + 0.7152 G + 0.0722 B$$ @@ -179,10 +203,15 @@ plt.show() Now, applying the [linalg.svd](https://numpy.org/devdocs/reference/generated/numpy.linalg.svd.html#numpy.linalg.svd) function to this matrix, we obtain the following decomposition: ```{code-cell} -U, s, Vt = linalg.svd(img_gray) +import numpy as np +U, s, Vt = np.linalg.svd(img_gray) ``` -**Note** If you are using your own image, this command might take a while to run, depending on the size of your image and your hardware. Don't worry, this is normal! The SVD can be a pretty intensive computation. +```{note} +If you are using your own image, this command might take a while to run, +depending on the size of your image and your hardware. +Don't worry, this is normal! The SVD can be a pretty intensive computation. +``` +++ @@ -193,17 +222,16 @@ U.shape, s.shape, Vt.shape ``` Note that `s` has a particular shape: it has only one dimension. This means that some linear algebra functions that expect 2d arrays might not work. For example, from the theory, one might expect `s` and `Vt` to be -compatible for multiplication. However, this is not true as `s` does not have a second axis. Executing +compatible for multiplication. However, this is not true as `s` does not have a second axis: -```python +```{code-cell} +:tags: [raises-exception] s @ Vt ``` results in a `ValueError`. This happens because having a one-dimensional array for `s`, in this case, is much more economic in practice than building a diagonal matrix with the same data. To reconstruct the original matrix, we can rebuild the diagonal matrix $\Sigma$ with the elements of `s` in its diagonal and with the appropriate dimensions for multiplying: in our case, $\Sigma$ should be 768x1024 since `U` is 768x768 and `Vt` is 1024x1024. In order to add the singular values to the diagonal of `Sigma`, we will use the [fill_diagonal](https://numpy.org/devdocs/reference/generated/numpy.fill_diagonal.html) function from NumPy: ```{code-cell} -import numpy as np - Sigma = np.zeros((U.shape[1], Vt.shape[0])) np.fill_diagonal(Sigma, s) ``` @@ -217,7 +245,7 @@ Now, we want to check if the reconstructed `U @ Sigma @ Vt` is close to the orig The [linalg](https://numpy.org/devdocs/reference/routines.linalg.html#module-numpy.linalg) module includes a `norm` function, which computes the norm of a vector or matrix represented in a NumPy array. For example, from the SVD explanation above, we would expect the norm of the difference between `img_gray` and the reconstructed SVD product to be small. As expected, you should see something like ```{code-cell} -linalg.norm(img_gray - U @ Sigma @ Vt) +np.linalg.norm(img_gray - U @ Sigma @ Vt) ``` (The actual result of this operation might be different depending on your architecture and linear algebra setup. Regardless, you should see a small number.) @@ -276,14 +304,10 @@ img_array.shape ``` so we need to permutate the axis on this array to get a shape like `(3, 768, 1024)`. Fortunately, the [numpy.transpose](https://numpy.org/devdocs/reference/generated/numpy.transpose.html#numpy.transpose) function can do that for us: -``` -np.transpose(x, axes=(i, j, k)) -``` -indicates that the axis will be reordered such that the final shape of the transposed array will be reordered according to the indices `(i, j, k)`. - -Let's see how this goes for our array: ```{code-cell} +# The values in the tuple indicate the original dim, and the order the new axis +# so axis 2 -> 0, 0 -> 1, and 1 -> 2 img_array_transposed = np.transpose(img_array, (2, 0, 1)) img_array_transposed.shape ``` @@ -291,7 +315,7 @@ img_array_transposed.shape Now we are ready to apply the SVD: ```{code-cell} -U, s, Vt = linalg.svd(img_array_transposed) +U, s, Vt = np.linalg.svd(img_array_transposed) ``` Finally, to obtain the full approximated image, we need to reassemble these matrices into the approximation. Now, note that @@ -342,7 +366,12 @@ plt.imshow(np.transpose(reconstructed, (1, 2, 0))) plt.show() ``` -In fact, `imshow` peforms this clipping under-the-hood, so if you skip the first line in the previous code cell, you might see a warning message saying `"Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers)."` +```{note} +In fact, `imshow` peforms this clipping under-the-hood, so if you skip the first +line in the previous code cell, you might see a warning message saying +`"Clipping input data to the valid range for imshow with RGB data ([0..1] for +floats or [0..255] for integers)."` +``` Now, to do the approximation, we must choose only the first `k` singular values for each color channel. This can be done using the following syntax: @@ -362,7 +391,7 @@ approx_img.shape which is not the right shape for showing the image. Finally, reordering the axes back to our original shape of `(768, 1024, 3)`, we can see our approximation: ```{code-cell} -plt.imshow(np.transpose(approx_img, (1, 2, 0))) +plt.imshow(np.transpose(np.clip(approx_img, 0, 1), (1, 2, 0))) plt.show() ``` @@ -379,6 +408,6 @@ terms of the norm of the difference. For more information, see *G. H. Golub and - [Python tutorial](https://docs.python.org/dev/tutorial/index.html) - [NumPy Reference](https://numpy.org/devdocs/reference/index.html#reference) -- [SciPy Tutorial](https://docs.scipy.org/doc/scipy/reference/tutorial/index.html) +- [SciPy Tutorial](https://docs.scipy.org/doc/scipy/tutorial/index.html) - [SciPy Lecture Notes](https://scipy-lectures.org) - [A matlab, R, IDL, NumPy/SciPy dictionary](http://mathesaurus.sf.net/) diff --git a/content/tutorial-x-ray-image-processing.md b/content/tutorial-x-ray-image-processing.md index c8dd3a2b..aa3f508a 100644 --- a/content/tutorial-x-ray-image-processing.md +++ b/content/tutorial-x-ray-image-processing.md @@ -187,7 +187,7 @@ notebook: ```{code-cell} ipython3 GIF_PATH = os.path.join(DIR, "xray_image.gif") -imageio.mimwrite(GIF_PATH, combined_xray_images_1, format= ".gif", fps=1) +imageio.mimwrite(GIF_PATH, combined_xray_images_1, format= ".gif", duration=1000) ``` Which gives us: @@ -523,6 +523,6 @@ simply edge detection, you may find the following material useful: - [Object detection with Raspberry Pi and Python](https://makersportal.com/blog/2019/4/23/image-processing-with-raspberry-pi-and-python-part-ii-spatial-statistics-and-correlations) (Maker Portal) - [X-ray data preparation and segmentation](https://www.kaggle.com/eduardomineo/u-net-lung-segmentation-montgomery-shenzhen) with deep learning (a Kaggle-hosted Jupyter notebook) - [Image filtering](https://www.cs.cornell.edu/courses/cs6670/2011sp/lectures/lec02_filter.pdf) (lecture slides, CS6670: Computer Vision, Cornell University) -- [Edge detection in Python](https://towardsdatascience.com/edge-detection-in-python-a3c263a13e03) and NumPy (Towards Data Science) -- [Edge detection](https://datacarpentry.org/image-processing/08-edge-detection/) with Scikit-Image (Data Carpentry) +- [Edge detection in Python](https://scikit-image.org/docs/0.25.x/auto_examples/edges/plot_edge_filter.html) +- [Edge detection](https://datacarpentry.github.io/image-processing/edge-detection.html) with Scikit-Image (Data Carpentry) - [Image gradients and gradient filtering](https://www.cs.cmu.edu/~16385/s17/Slides/4.0_Image_Gradients_and_Gradient_Filtering.pdf) (lecture slides, 16-385 Computer Vision, Carnegie Mellon University) diff --git a/content/x_y-squared.csv b/content/x_y-squared.csv deleted file mode 100644 index e74126ff..00000000 --- a/content/x_y-squared.csv +++ /dev/null @@ -1,11 +0,0 @@ -# x, y -0.000000000000000000e+00,0.000000000000000000e+00 -1.000000000000000000e+00,1.000000000000000000e+00 -2.000000000000000000e+00,4.000000000000000000e+00 -3.000000000000000000e+00,9.000000000000000000e+00 -4.000000000000000000e+00,1.600000000000000000e+01 -5.000000000000000000e+00,2.500000000000000000e+01 -6.000000000000000000e+00,3.600000000000000000e+01 -7.000000000000000000e+00,4.900000000000000000e+01 -8.000000000000000000e+00,6.400000000000000000e+01 -9.000000000000000000e+00,8.100000000000000000e+01 diff --git a/content/x_y-squared.npz b/content/x_y-squared.npz deleted file mode 100644 index 6c32f196..00000000 Binary files a/content/x_y-squared.npz and /dev/null differ diff --git a/environment.yml b/environment.yml index 1a8abe27..769fb7b0 100644 --- a/environment.yml +++ b/environment.yml @@ -5,12 +5,14 @@ dependencies: # For running the tutorials - numpy - scipy + - pooch - matplotlib - - pandas - - statsmodels + - pandas - imageio # For building the site - - sphinx<5 - - myst-nb - - sphinx-book-theme - - sphinx-copybutton + - tox + - jupytext + - pip + - pip: + - jupyter-book>=2 + - jupyterlab_myst diff --git a/ignore_testing b/ignore_testing new file mode 100644 index 00000000..68762fe8 --- /dev/null +++ b/ignore_testing @@ -0,0 +1 @@ +content/tutorial-style-guide.md diff --git a/myst.yml b/myst.yml new file mode 100644 index 00000000..fa935f15 --- /dev/null +++ b/myst.yml @@ -0,0 +1,35 @@ +version: 1 +project: + title: Numpy Tutorials + # description: + # keywords: [] + authors: Numpy Community + github: https://github.com/numpy/numpy-tutorials + toc: + - file: site/index.md + + - title: Applications + children: + - file: content/mooreslaw-tutorial.md + - file: content/tutorial-deep-learning-on-mnist.md + - file: content/tutorial-x-ray-image-processing.md + - file: content/tutorial-static_equilibrium.md + - file: content/tutorial-plotting-fractals.md + - file: content/tutorial-air-quality-analysis.md + + - title: Features + children: + - file: content/tutorial-svd.md + - file: content/save-load-arrays.md + - file: content/tutorial-ma.md + - title: Contributing + file: site/contributing.md + children: + - file: content/tutorial-style-guide.md + + +site: + template: book-theme + options: + favicon: site/_static/favicon.png + logo: site/_static/numpylogo.svg diff --git a/requirements.txt b/requirements.txt index 63b2eefa..dd03cc89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ # For the tutorials numpy scipy +pooch # for scipy.datasets matplotlib pandas -statsmodels imageio # For supporting .md-based notebooks jupytext diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 00000000..55090899 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.10 diff --git a/site/Makefile b/site/Makefile deleted file mode 100644 index 89fe53b6..00000000 --- a/site/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -notebooks: - mkdir -p notebooks - jupytext -k python3 ../content/*.md --from myst --to notebook - mv ../content/*.ipynb notebooks - -clean: - rm -rf _build - rm -rf notebooks - -.PHONY: help Makefile notebooks clean - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/site/applications.md b/site/applications.md deleted file mode 100644 index 743143b0..00000000 --- a/site/applications.md +++ /dev/null @@ -1,19 +0,0 @@ -# NumPy Applications - -A collection of highlighting the use of NumPy for applications in science, -engineering, and data analysis. - -```{toctree} ---- -maxdepth: 1 ---- - -content/mooreslaw-tutorial -content/tutorial-deep-learning-on-mnist -content/tutorial-deep-reinforcement-learning-with-pong-from-pixels -content/tutorial-nlp-from-scratch -content/tutorial-x-ray-image-processing -content/tutorial-static_equilibrium -content/tutorial-plotting-fractals -content/tutorial-air-quality-analysis -``` diff --git a/site/conf.py b/site/conf.py index b9ab2a01..881b3ab2 100644 --- a/site/conf.py +++ b/site/conf.py @@ -12,9 +12,10 @@ # -- Project information ----------------------------------------------------- +from datetime import date project = 'NumPy tutorials' -copyright = '2020, the NumPy community' +copyright = f'2020-{date.today().year}, the NumPy community' author = 'the NumPy community' @@ -42,11 +43,10 @@ 'Thumbs.db', '.DS_Store', 'notebooks', - 'content/tutorial-nlp-from-scratch/README.md', ] # MyST-NB configuration -nb_execution_timeout = 900 +nb_execution_timeout = 60 # -- Options for HTML output ------------------------------------------------- @@ -64,7 +64,7 @@ "repository_branch": "main", "use_repository_button": True, "use_issues_button": True, - "use_edit_page_button": True, + "use_edit_page_button": False, "path_to_docs": "site/", "launch_buttons": { "binderhub_url": "https://mybinder.org", diff --git a/site/contributing.md b/site/contributing.md index 8985c56c..78caf34e 100644 --- a/site/contributing.md +++ b/site/contributing.md @@ -1,127 +1,2 @@ -# Contributing - -We very much welcome contributions! If you have an idea or proposal for a new -tutorial, please [open an issue](https://github.com/numpy/numpy-tutorials/issues) -with an outline. - -Don’t worry if English is not your first language, or if you can only come up -with a rough draft. Open source is a community effort. Do your best – we’ll help -fix issues. - -Images and real-life data make text more engaging and powerful, but be sure what -you use is appropriately licensed and available. Here again, even a rough idea -for artwork can be polished by others. - -The NumPy tutorials are a curated collection of -[MyST-NB](https://myst-nb.readthedocs.io/) notebooks. These notebooks are used -to produce static websites and can be opened as notebooks in Jupyter using -[Jupytext](https://jupytext.readthedocs.io). - -> __Note:__ You should use [CommonMark](https://commonmark.org) markdown -> cells. Jupyter only renders CommonMark. - -## Why Jupyter Notebooks? - -The choice of Jupyter Notebook in this repo instead of the usual format -([reStructuredText][rst]) -used in the main NumPy documentation has two reasons: - - - * Jupyter notebooks are a common format for communicating scientific - information. - * Jupyter notebooks can be launched in [Binder](https://www.mybinder.org), so that users can interact - with tutorials - * rST may present a barrier for some people who might otherwise be very - interested in contributing tutorial material. - -[rst]: https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html - -### Note - -You may notice our content is in markdown format (`.md` files). We review and -host notebooks in the [MyST-NB](https://myst-nb.readthedocs.io/) format. We -accept both Jupyter notebooks (`.ipynb`) and MyST-NB notebooks (`.md`). -If you want to sync your `.ipynb` to your `.md` file follow the [pairing -tutorial](content/pairing.md). - -```{toctree} -:hidden: - -content/pairing +```{include} ../CONTRIBUTING.md ``` - -## Adding your own tutorials - -If you have your own tutorial in the form of a Jupyter notebook (an `.ipynb` -file) and you'd like to try add it out to the repository, follow the steps below. - -### Create an issue - -Go to and create a new issue -with your proposal. -Give as much detail as you can about what kind of content you would like to -write (tutorial, how-to) and what you plan to cover. -We will try to respond as quickly as possible with comments, if applicable. - -### Check out our suggested template - -You can use this template to make your content consistent with our existing -tutorials: - -```{toctree} ---- -maxdepth: 1 ---- -content/tutorial-style-guide -``` - -### Upload your content - -Remember to clear all outputs on your notebook before uploading it. - -
    -
    - - Fork this repository (if you haven't before). - - -
    - -
    - - In your own fork, create a new branch for your content. - - -
    - -
    - - Add your notebook to the content/ directory. - - -
    - -Update the environment.yml file with the dependencies for your tutorial -(only if you add new dependencies). - -
    - - Update this README.md to include your new entry. - - -
    - -
    - - Create a pull request. Make sure the "Allow edits and access to secrets by maintainers" option is selected so we can properly review your submission. - - -
    - -🎉 Wait for review! -
- -For more information about GitHub and its workflow, you can see -[this document][collab]. - -[collab]: https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests diff --git a/site/features.md b/site/features.md deleted file mode 100644 index fb966a8a..00000000 --- a/site/features.md +++ /dev/null @@ -1,13 +0,0 @@ -# NumPy Features - -A collection of notebooks pertaining to built-in NumPy functionality. - -```{toctree} ---- -maxdepth: 1 ---- - -content/tutorial-svd -content/save-load-arrays -content/tutorial-ma -``` diff --git a/site/index.md b/site/index.md index 6d6baa3a..b026eacc 100644 --- a/site/index.md +++ b/site/index.md @@ -19,18 +19,6 @@ local copy of the `.ipynb` files, you can either [clone this repository](https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/cloning-a-repository) or use the download icon in the upper-right corner of each tutorial. -## Content - -```{toctree} ---- -maxdepth: 2 ---- - -features -applications -contributing -``` - ## Useful links and resources The following links may be useful: diff --git a/site/requirements.txt b/site/requirements.txt index e040deba..e6e749ac 100644 --- a/site/requirements.txt +++ b/site/requirements.txt @@ -1,4 +1 @@ -sphinx -myst-nb -sphinx-book-theme -sphinx-copybutton +jupyter-book>=2 diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..d20d5b24 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,2 @@ +pytest +nbval diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..8ec29880 --- /dev/null +++ b/tox.ini @@ -0,0 +1,52 @@ +[tox] +envlist = + py{310,311,312,313,314}-test{,-oldestdeps,-devdeps,-predeps}{,-buildhtml} +requires = + pip >= 19.3.1 + +[testenv] + +description = run tests + +setenv = + devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/scientific-python-nightly-wheels/simple + +passenv = BASE_URL, OMP_NUM_THREADS + +deps = + # We use these files to specify all the dependencies, and below we override + # versions for specific testing schenarios + -rtest_requirements.txt + -rsite/requirements.txt + -rrequirements.txt + + oldestdeps: numpy==1.24 + oldestdeps: matplotlib==3.7 + oldestdeps: scipy==1.10 + oldestdeps: pandas==2.0 + + devdeps: numpy>=0.0.dev0 + devdeps: scipy>=0.0.dev0 + devdeps: matplotlib>=0.0.dev0 + devdeps: pandas>=0.0.dev0 + +allowlist_externals = bash + +commands = + # Force numpy reinstall to work around upper version limits in downstream dependencies (e.g. pandas) + devdeps: pip install -U --pre --no-deps --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy + + pip freeze + + # Ignore testing the tutorials listed in ignore_testing file + !buildhtml: bash -c 'find content -name "*.md" | grep -vf ignore_testing | xargs jupytext --to notebook ' + + !buildhtml: pytest --nbval-lax --durations=10 content/ + + buildhtml: bash -c "jupyter-book build --execute --html -d" + +pip_pre = + predeps: true + !predeps: false + +skip_install = true