Merge branch 'development' into 'master'

Development

See merge request !18

Dockerfile

 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/opensource/jupyter/jupyterlab
-ARG BASE_TAG=3.0.14
+ARG BASE_TAG=3.0.16
 
 FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
 
@@ -19,11 +19,10 @@ USER ${USER}
 RUN mkdir /envs/jaic-df && \
     mkdir /envs/jaic-mf 
 
-COPY envs/default/requirements.txt /tmp/
 COPY --chown=$USER envs/jaic-df/* /envs/jaic-df/
 COPY --chown=$USER envs/jaic-mf/* /envs/jaic-mf/
 
-RUN python -m pip install -U --no-deps --no-index -f /tmp/packages/ -r /tmp/requirements.txt && \
+RUN python -m pip install --no-index -f /tmp/packages/ poetry && \
     cd /envs && \
     cd /envs/jaic-df && \
     poetry env use /usr/local/bin/python && \
@@ -41,7 +40,14 @@ USER root
 
 RUN rm -f /home/jovyan/.cache/pypoetry/virtualenvs/jaic-df-qX42fgRL-py3.8/lib/python3.8/site-packages/featuretools/tests/primitive_tests/test_feature_serialization.py && \
     rm -f /home/jovyan/.cache/pypoetry/virtualenvs/jaic-df-qX42fgRL-py3.8/lib/python3.8/site-packages/featuretools/tests/primitive_tests/__pycache__/test_feature_serialization.cpython-38.pyc && \
+    rm -rf /home/jovyan/.cache/pypoetry/virtualenvs/jaic-mf-t_BZc04u-py3.8/lib/python3.8/site-packages/bleach/_vendor/Django-1.11.29.dist-info && \
+    rm -rf /home/jovyan/.cache/pypoetry/virtualenvs/jaic-mf-t_BZc04u-py3.8/lib/python3.8/site-packages/bleach/_vendor/django && \
     find /home/jovyan/.cache/pypoetry/virtualenvs -name "*.pem" -o -name "*.key" | grep -v "certifi/cacert.pem" | xargs rm -f && \
+    mv /home/jovyan/.cache /tmp/ && \
+    mv /home/jovyan/.local /tmp/ && \
+    ln -s /tmp/.cache /home/jovyan/.cache && \
+    ln -s /tmp/.local /home/jovyan/.local && \
+
     dnf remove -y xz-devel make gcc gcc-c++ openssl-devel bzip2-devel libffi-devel sqlite-devel && \
     rm -rf /tmp/* /tmp/packages/ /opt/jupyter/*.whl /opt/jupyter/*.tar.gz
 

envs/default/requirements-top-level.txt

-pip
-jupyterlab
+poetryS
\ No newline at end of file
-jupyterlab-git
-poetry
\ No newline at end of file

envs/default/requirements.txt

+anyio==3.2.1
 appdirs==1.4.4
+argon2-cffi==20.1.0
+async-generator==1.10
+attrs==21.2.0
+Babel==2.9.1
+backcall==0.2.0
+bleach==3.3.0
 CacheControl==0.12.6
 cachy==0.3.0
+certifi==2021.5.30
+cffi==1.14.5
+chardet==4.0.0
 cleo==0.8.1
 clikit==0.6.2
+colorama==0.4.4
 crashtest==0.3.1
-distlib==0.3.1
+cryptography==3.4.7
+debugpy==1.3.0
+decorator==5.0.9
+defusedxml==0.7.1
+distlib==0.3.2
+entrypoints==0.3
 filelock==3.0.12
+gitdb==4.0.7
+GitPython==3.1.18
 html5lib==1.1
+idna==2.10
+ipykernel==6.0.1
+ipython==7.25.0
+ipython-genutils==0.2.0
+jedi==0.18.0
 jeepney==0.6.0
+Jinja2==3.0.1
+json5==0.9.6
+jsonschema==3.2.0
+jupyter-client==6.1.12
+jupyter-core==4.7.1
+jupyter-server==1.9.0
+jupyter-server-mathjax==0.2.3
+jupyterlab==3.0.16
+jupyterlab-git==0.30.1
+jupyterlab-pygments==0.1.2
+jupyterlab-server==2.6.0
 keyring==21.8.0
 lockfile==0.12.2
+MarkupSafe==2.0.1
+matplotlib-inline==0.1.2
+mistune==0.8.4
 msgpack==1.0.2
+nbclassic==0.3.1
+nbclient==0.5.3
+nbconvert==6.1.0
+nbdime==3.1.0
+nbformat==5.1.3
+nest-asyncio==1.5.1
+notebook==6.4.0
+packaging==20.9
+pandocfilters==1.4.3
+parso==0.8.2
 pastel==0.2.1
+pexpect==4.8.0
+pickleshare==0.7.5
+pip==21.1.3
 pkginfo==1.7.0
-poetry==1.1.6
+poetry==1.1.7
 poetry-core==1.0.3
-pylev==1.3.0
+prometheus-client==0.11.0
+prompt-toolkit==3.0.19
+ptyprocess==0.7.0
+pycparser==2.20
+Pygments==2.9.0
+pylev==1.4.0
+pyparsing==2.4.7
+pyrsistent==0.18.0
+python-dateutil==2.8.1
+pytz==2021.1
+pyzmq==22.1.0
+requests==2.25.1
 requests-toolbelt==0.9.1
+requests-unixsocket==0.2.0
 SecretStorage==3.3.1
+Send2Trash==1.7.1
+setuptools==56.1.0
 shellingham==1.4.0
-virtualenv==20.4.6
+six==1.16.0
\ No newline at end of file
+smmap==4.0.0
+sniffio==1.2.0
+terminado==0.10.1
+testpath==0.5.0
+tomlkit==0.7.2
+tornado==6.1
+traitlets==5.0.5
+urllib3==1.26.6
+virtualenv==20.4.7
+wcwidth==0.2.5
+webencodings==0.5.1
+websocket-client==1.1.0
+wheel==0.36.2

envs/jaic-df/requirements-top-level.txt

 streamz
 featuretools
 snorkel
-tsfresh
+tsfresh
\ No newline at end of file
+nltk
+spacy
+textblob
\ No newline at end of file

envs/jaic-df/requirements.txt

-absl-py==0.12.0
+absl-py==0.13.0
 attrs==21.2.0
 backcall==0.2.0
 blis==0.7.4
 catalogue==2.0.4
-certifi==2020.12.5
+certifi==2021.5.30
 chardet==4.0.0
 click==7.1.2
 cloudpickle==1.6.0
 cycler==0.10.0
 cymem==2.0.5
-dask==2021.5.0
+dask==2021.6.2
 decorator==5.0.9
-distributed==2021.5.0
+distributed==2021.6.2
 featuretools==0.24.1
-fsspec==2021.5.0
-grpcio==1.38.0
+fsspec==2021.6.1
+grpcio==1.38.1
 HeapDict==1.0.1
 idna==2.10
 ipykernel==5.5.5
-ipython==7.23.1
+ipython==7.25.0
 ipython-genutils==0.2.0
 jedi==0.18.0
 Jinja2==3.0.1
@@ -42,24 +42,24 @@ networkx==2.3
 nltk==3.6.2
 numba==0.53.1
 numpy==1.19.5
-packaging==20.9
-pandas==1.2.4
+packaging==21.0
+pandas==1.3.0
 parso==0.8.2
 partd==1.2.0
-pathy==0.5.2
+pathy==0.6.0
 patsy==0.5.1
 pexpect==4.8.0
 pickleshare==0.7.5
-Pillow==8.2.0
+Pillow==8.3.1
 pip==21.1.1
 pluggy==0.13.1
 preshed==3.0.5
-prompt-toolkit==3.0.18
+prompt-toolkit==3.0.19
 protobuf==3.11.2
 psutil==5.8.0
 ptyprocess==0.7.0
 py==1.10.0
-pydantic==1.7.4
+pydantic==1.8.2
 Pygments==2.9.0
 pyparsing==2.4.7
 pytest==5.4.3
@@ -67,17 +67,17 @@ python-dateutil==2.8.1
 pytz==2021.1
 PyYAML==5.4.1
 pyzmq==22.1.0
-regex==2021.4.4
+regex==2021.7.6
 requests==2.25.1
 scikit-learn==0.24.2
-scipy==1.6.3
+scipy==1.7.0
 setuptools==56.0.0
 six==1.16.0
-smart-open==3.0.0
+smart-open==5.1.0
 snorkel==0.9.7
 sortedcontainers==2.4.0
-spacy==3.0.6
-spacy-legacy==3.0.5
+spacy==3.1.0
+spacy-legacy==3.0.8
 srsly==2.4.1
 statsmodels==0.12.2
 streamz==0.6.2
@@ -85,17 +85,17 @@ stumpy==1.8.0
 tblib==1.7.0
 tensorboard==1.15.0
 textblob==0.15.3
-thinc==8.0.3
+thinc==8.0.7
 threadpoolctl==2.1.0
 toolz==0.11.1
-torch==1.8.1
+torch==1.9.0
 tornado==6.1
-tqdm==4.61.0
+tqdm==4.61.2
 traitlets==5.0.5
 tsfresh==0.18.0
 typer==0.3.2
 typing-extensions==3.10.0.0
-urllib3==1.26.5
+urllib3==1.26.6
 wasabi==0.8.2
 wcwidth==0.2.5
 Werkzeug==2.0.1

envs/jaic-mf/pyproject.toml

@@ -27,6 +27,7 @@ nltk = {version = "^3.6.2", python = ">=3.8,<3.9"}
 spacy = {version = "^3.0.6", python = ">=3.8,<3.9"}
 textblob = {version = "^0.15.3", python = ">=3.8,<3.9"}
 ipykernel = {version = "^5.5.5", python = ">=3.8,<3.9"}
+tensorflow-data-validation = {version = "^1.1.0", python = ">=3.8,<3.9"}
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"

envs/jaic-mf/requirements.txt

 absl-py==0.12.0
 ansiwrap==0.8.4
-apache-beam==2.29.0
+apache-beam==2.30.0
 appdirs==1.4.4
 argon2-cffi==20.1.0
 astunparse==1.6.3
 async-generator==1.10
 attrs==20.3.0
 avro-python3==1.9.2.1
-azure-core==1.14.0
+azure-core==1.16.0
 azure-storage-blob==12.8.1
 backcall==0.2.0
 black==20.8b1
@@ -15,7 +15,7 @@ bleach==3.3.0
 blis==0.7.4
 cachetools==4.2.2
 catalogue==2.0.4
-certifi==2020.12.5
+certifi==2021.5.30
 cffi==1.14.5
 chardet==4.0.0
 click==7.1.2
@@ -26,7 +26,7 @@ cryptography==3.4.7
 cycler==0.10.0
 cymem==2.0.5
 Cython==0.29.23
-dask==2021.5.0
+dask==2021.6.2
 decorator==5.0.9
 defusedxml==0.7.1
 dill==0.3.1.1
@@ -34,45 +34,45 @@ dm-tree==0.1.6
 docker==4.4.4
 docopt==0.6.2
 entrypoints==0.3
-fastavro==1.4.1
-fasteners==0.16
+fastavro==1.4.2
+fasteners==0.16.3
 flatbuffers==1.12
-fsspec==2021.5.0
+fsspec==2021.6.1
 future==0.18.2
 gast==0.4.0
 gin-config==0.4.0
-google-api-core==1.28.0
+google-api-core==1.30.0
 google-api-python-client==1.12.8
 google-apitools==0.5.31
-google-auth==1.30.1
+google-auth==1.32.1
 google-auth-httplib2==0.1.0
 google-auth-oauthlib==0.4.4
 google-cloud-aiplatform==0.7.1
-google-cloud-bigquery==1.28.0
+google-cloud-bigquery==2.20.0
 google-cloud-bigtable==1.7.0
-google-cloud-core==1.6.0
+google-cloud-core==1.7.1
 google-cloud-datastore==1.15.3
 google-cloud-dlp==1.0.0
 google-cloud-language==1.3.0
 google-cloud-pubsub==1.7.0
 google-cloud-spanner==1.19.1
-google-cloud-storage==1.38.0
+google-cloud-storage==1.40.0
 google-cloud-videointelligence==1.16.1
 google-cloud-vision==1.0.0
 google-crc32c==1.1.2
 google-pasta==0.2.0
-google-resumable-media==1.3.0
+google-resumable-media==1.3.1
 googleapis-common-protos==1.53.0
 grpc-google-iam-v1==0.12.3
 grpcio==1.34.1
 grpcio-gcp==0.2.2
 h5py==3.1.0
 hdfs==2.6.0
-httplib2==0.17.4
+httplib2==0.19.1
 idna==2.10
-importlib-resources==5.1.4
+importlib-resources==5.2.0
 ipykernel==5.5.5
-ipython==7.23.1
+ipython==7.25.0
 ipython-genutils==0.2.0
 ipywidgets==7.6.3
 isodate==0.6.0
@@ -98,44 +98,44 @@ MarkupSafe==2.0.1
 matplotlib==3.4.2
 matplotlib-inline==0.1.2
 mistune==0.8.4
-ml-metadata==1.1.0.dev20210526
+ml-metadata==1.1.0.dev20210707
 ml-pipelines-sdk==0.31.0.dev20210519
 more-itertools==8.8.0
 msrest==0.6.21
 murmurhash==1.0.5
 mypy-extensions==0.4.3
 nbclient==0.5.3
-nbconvert==6.0.7
+nbconvert==6.1.0
 nbformat==5.1.3
 nest-asyncio==1.5.1
 nltk==3.6.2
 notebook==6.4.0
 numpy==1.19.5
 oauth2client==4.1.3
-oauthlib==3.1.0
-opencv-python-headless==4.5.2.52
+oauthlib==3.1.1
+opencv-python-headless==4.5.2.54
 opt-einsum==3.3.0
 packaging==20.9
-pandas==1.2.4
+pandas==1.3.0
 pandocfilters==1.4.3
 papermill==2.3.3
 parso==0.8.2
 partd==1.2.0
 pathspec==0.8.1
-pathy==0.5.2
+pathy==0.6.0
 pexpect==4.8.0
 pickleshare==0.7.5
-Pillow==8.2.0
+Pillow==8.3.1
 pip==21.1.1
 pluggy==0.13.1
 portalocker==2.0.0
 portpicker==1.4.0
 preshed==3.0.5
-prometheus-client==0.10.1
+prometheus-client==0.11.0
 promise==2.3
-prompt-toolkit==3.0.18
-proto-plus==1.18.1
-protobuf==3.17.1
+prompt-toolkit==3.0.19
+proto-plus==1.19.0
+protobuf==3.17.3
 psutil==5.8.0
 ptyprocess==0.7.0
 py==1.10.0
@@ -145,34 +145,34 @@ pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycocotools==2.0.2
 pycparser==2.20
-pydantic==1.7.4
+pydantic==1.8.2
 pydot==1.4.2
 Pygments==2.9.0
 pymongo==3.11.4
 pyparsing==2.4.7
-pyrsistent==0.17.3
+pyrsistent==0.18.0
 pytest==5.4.3
 python-dateutil==2.8.1
 python-slugify==5.0.2
 pytz==2021.1
 PyYAML==5.4.1
 pyzmq==22.1.0
-regex==2021.4.4
+regex==2021.7.6
 requests==2.25.1
 requests-oauthlib==1.3.0
 rsa==4.7.2
 sacrebleu==1.5.1
 scikit-learn==0.24.2
-scipy==1.6.3
-Send2Trash==1.5.0
-sentencepiece==0.1.95
+scipy==1.7.0
+Send2Trash==1.7.1
+sentencepiece==0.1.96
 seqeval==1.2.2
 setuptools==56.0.0
 six==1.15.0
 sklearn==0.0
-smart-open==3.0.0
-spacy==3.0.6
-spacy-legacy==3.0.5
+smart-open==5.1.0
+spacy==3.1.0
+spacy-legacy==3.0.8
 srsly==2.4.1
 tabulate==0.8.9
 tenacity==7.0.0
@@ -181,19 +181,19 @@ tensorboard-data-server==0.6.1
 tensorboard-plugin-wit==1.8.0
 tensorflow==2.5.0
 tensorflow-addons==0.13.0
-tensorflow-data-validation==1.1.0.dev20210526
+tensorflow-data-validation==1.1.0
 tensorflow-datasets==4.3.0
 tensorflow-estimator==2.5.0
 tensorflow-hub==0.9.0
-tensorflow-metadata==1.1.0.dev20210526
-tensorflow-model-analysis==0.32.0.dev20210526
-tensorflow-model-optimization==0.5.0
+tensorflow-metadata==1.1.0
+tensorflow-model-analysis==0.32.0
+tensorflow-model-optimization==0.6.0
 tensorflow-ranking==0.3.3
 tensorflow-serving-api==2.5.1
 tensorflow-text==2.5.0rc0
-tensorflow-transform==1.1.0.dev20210526
+tensorflow-transform==1.1.0
 termcolor==1.1.0
-terminado==0.10.0
+terminado==0.10.1
 terminaltables==3.1.0
 testpath==0.5.0
 text-unidecode==1.3
@@ -203,26 +203,26 @@ tf-models-official==2.5.0
 tf-slim==1.1.0
 tf-utils==1.0.4
 tfx==0.31.0.dev20210519
-tfx-bsl==1.1.0.dev20210526
-thinc==8.0.3
+tfx-bsl==1.1.0
+thinc==8.0.7
 threadpoolctl==2.1.0
 toml==0.10.2
 toolz==0.11.1
 tornado==6.1
-tqdm==4.61.0
+tqdm==4.61.2
 traitlets==5.0.5
 typed-ast==1.4.3
-typeguard==2.12.0
+typeguard==2.12.1
 typer==0.3.2
 typing-extensions==3.7.4.3
 uritemplate==3.0.1
-urllib3==1.26.5
+urllib3==1.26.6
 wasabi==0.8.2
 wcwidth==0.2.5
 webencodings==0.5.1
-websocket-client==1.0.1
+websocket-client==1.1.0
 Werkzeug==2.0.1
 wheel==0.36.2
 widgetsnbextension==3.5.1
 wrapt==1.12.1
-zipp==3.4.1
+zipp==3.5.0